diff mbox series

[v3,1/1] package/fluentbit: new package

Message ID 20200217145805.18815-1-frank.vanbever@essensium.com
State Changes Requested
Headers show
Series [v3,1/1] package/fluentbit: new package | expand

Commit Message

Frank Vanbever Feb. 17, 2020, 2:58 p.m. UTC 
From: Kevin Hilman <khilman@baylibre.com>

Fluent Bit is an open source and multi-platform Log Processor and
Forwarder which allows you to collect data/logs from different
sources, unify and send them to multiple destinations.

_FILE_OFFSET_BITS==64 needs to be filtered out to allow building with old glibc
versions (<2.23).

0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch is
required to allow building for PowerPC.

Signed-off-by: Kevin Hilman <khilman@baylibre.com>
Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>
---
v2 -> v3:
 - Cleaned up included patch: "Backported from:", "Signed off by:" and some
   additional explanation on why the patch is necessary

This patch is adapted from http://patchwork.ozlabs.org/patch/1038388/
with some clean-up and additional fixes.

Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>
---
 DEVELOPERS                                    |  1 +
 package/Config.in                             |  1 +
 ...c-Fix-signature-for-co_create-API-18.patch | 42 ++++++++++++
 package/fluent-bit/Config.in                  | 15 +++++
 package/fluent-bit/fluent-bit.hash            |  3 +
 package/fluent-bit/fluent-bit.mk              | 65 +++++++++++++++++++
 6 files changed, 127 insertions(+)
 create mode 100644 package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch
 create mode 100644 package/fluent-bit/Config.in
 create mode 100644 package/fluent-bit/fluent-bit.hash
 create mode 100644 package/fluent-bit/fluent-bit.mk

Comments

Romain Naour March 24, 2020, 9:42 p.m. UTC | #1 
Le 17/02/2020 à 15:58, Frank Vanbever a écrit :
> From: Kevin Hilman <khilman@baylibre.com>
> 
> Fluent Bit is an open source and multi-platform Log Processor and
> Forwarder which allows you to collect data/logs from different
> sources, unify and send them to multiple destinations.
> 
> _FILE_OFFSET_BITS==64 needs to be filtered out to allow building with old glibc
> versions (<2.23).
> 
> 0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch is
> required to allow building for PowerPC.
> 
> Signed-off-by: Kevin Hilman <khilman@baylibre.com>
> Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>
> ---
> v2 -> v3:
>  - Cleaned up included patch: "Backported from:", "Signed off by:" and some
>    additional explanation on why the patch is necessary
> 
> This patch is adapted from http://patchwork.ozlabs.org/patch/1038388/
> with some clean-up and additional fixes.
> 
> Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>
> ---
>  DEVELOPERS                                    |  1 +
>  package/Config.in                             |  1 +
>  ...c-Fix-signature-for-co_create-API-18.patch | 42 ++++++++++++
>  package/fluent-bit/Config.in                  | 15 +++++
>  package/fluent-bit/fluent-bit.hash            |  3 +
>  package/fluent-bit/fluent-bit.mk              | 65 +++++++++++++++++++
>  6 files changed, 127 insertions(+)
>  create mode 100644 package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch
>  create mode 100644 package/fluent-bit/Config.in
>  create mode 100644 package/fluent-bit/fluent-bit.hash
>  create mode 100644 package/fluent-bit/fluent-bit.mk
> 
> diff --git a/DEVELOPERS b/DEVELOPERS
> index e4a3bcb24a..0c309d94e4 100644
> --- a/DEVELOPERS
> +++ b/DEVELOPERS
> @@ -960,6 +960,7 @@ F:	package/zxing-cpp/
>  
>  N:	Frank Vanbever <frank.vanbever@essensium.com>
>  F: 	package/elixir/
> +F: 	package/fluent-bit/
>  F:	package/libmodsecurity/
>  F:	package/nginx-modsecurity/
>  
> diff --git a/package/Config.in b/package/Config.in
> index dfa3f34b9d..f3d84f16e5 100644
> --- a/package/Config.in
> +++ b/package/Config.in
> @@ -2298,6 +2298,7 @@ menu "System tools"
>  	source "package/efibootmgr/Config.in"
>  	source "package/efivar/Config.in"
>  	source "package/emlog/Config.in"
> +	source "package/fluent-bit/Config.in"
>  	source "package/ftop/Config.in"
>  	source "package/getent/Config.in"
>  	source "package/htop/Config.in"
> diff --git a/package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch b/package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch
> new file mode 100644
> index 0000000000..c85bf85d33
> --- /dev/null
> +++ b/package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch
> @@ -0,0 +1,42 @@
> +From 731dab51a41d3635ceae5481d89d30148caac96d Mon Sep 17 00:00:00 2001
> +From: Khem Raj <raj.khem@gmail.com>
> +Date: Tue, 11 Feb 2020 19:36:21 -0800
> +Subject: [PATCH] lib: flb_libco: ppc: Fix signature for co_create API (#1886)
> +
> +This patch fixes build errors for the flb_libco dependency when builidng
> +for PowerPC.
> +
> +Backported from: 731dab51a41d3635ceae5481d89d30148caac96d
> +Signed-off-by: Khem Raj <raj.khem@gmail.com>
> +Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>
> +---
> + lib/flb_libco/ppc.c | 6 ++++--
> + 1 file changed, 4 insertions(+), 2 deletions(-)
> +
> +diff --git a/lib/flb_libco/ppc.c b/lib/flb_libco/ppc.c
> +index e6536d56..533256b3 100644
> +--- a/lib/flb_libco/ppc.c
> ++++ b/lib/flb_libco/ppc.c
> +@@ -279,7 +279,9 @@ static uint32_t* co_create_(unsigned size, uintptr_t entry) {
> +   return t;
> + }
> + 
> +-cothread_t co_create(unsigned int size, void (*entry_)(void)) {
> ++cothread_t co_create(unsigned int size, void (*entry_)(void),
> ++                     size_t *out_size) {
> ++
> +   uintptr_t entry = (uintptr_t)entry_;
> +   uint32_t* t = 0;
> + 
> +@@ -325,7 +327,7 @@ cothread_t co_create(unsigned int size, void (*entry_)(void)) {
> +     t[10] = (uint32_t)(sp >> shift >> shift);
> +     t[11] = (uint32_t)sp;
> +   }
> +-
> ++  *out_size = size;
> +   return t;
> + }
> + 
> +-- 
> +2.20.1
> +
> diff --git a/package/fluent-bit/Config.in b/package/fluent-bit/Config.in
> new file mode 100644
> index 0000000000..98efaa36ff
> --- /dev/null
> +++ b/package/fluent-bit/Config.in
> @@ -0,0 +1,15 @@
> +config BR2_PACKAGE_FLUENT_BIT
> +	bool "fluent-bit"
> +	depends on BR2_INSTALL_LIBSTDCPP
> +	depends on !BR2_STATIC_LIBS
> +	select BR2_PACKAGE_MUSL_FTS if !BR2_TOOLCHAIN_USES_GLIBC
> +	help
> +	  Fluent Bit is an open source and multi-platform Log
> +	  processor and forwarder which allows you to collect
> +	  data/logs from different sources, unify and send them to
> +	  multiple destinations.
> +
> +	  http://fluentbit.io
> +
> +comment "fluent-bit needs a toolchain w/ C++, dynamic library"
> +	depends on !BR2_INSTALL_LIBSTDCPP || BR2_STATIC_LIBS
> diff --git a/package/fluent-bit/fluent-bit.hash b/package/fluent-bit/fluent-bit.hash
> new file mode 100644
> index 0000000000..1a8e0d121c
> --- /dev/null
> +++ b/package/fluent-bit/fluent-bit.hash
> @@ -0,0 +1,3 @@
> +# computed locally
> +sha256  e897949356855e2a7d1a71085c7c21653b7792ef137d19d665e7b5dcd8e4d46a  fluent-bit-1.3.7.tar.gz
> +sha256  0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594  LICENSE
> diff --git a/package/fluent-bit/fluent-bit.mk b/package/fluent-bit/fluent-bit.mk
> new file mode 100644
> index 0000000000..c48d77a1fe
> --- /dev/null
> +++ b/package/fluent-bit/fluent-bit.mk
> @@ -0,0 +1,65 @@
> +################################################################################
> +#
> +# fluent-bit
> +#
> +################################################################################
> +
> +FLUENT_BIT_VERSION = 1.3.7
> +FLUENT_BIT_SITE = $(call github,fluent,fluent-bit,v$(FLUENT_BIT_VERSION))
> +FLUENT_BIT_LICENSE = Apache-2.0
> +FLUENT_BIT_LICENSE_FILE = LICENSE
> +
> +# LuaJIT and filter_lua do not support cross compilation
> +FLUENT_BIT_CONF_OPTS += -DFLB_LUAJIT=OFF -DFLB_FILTER_LUA=OFF
> +
> +# Only build the daemon. Disable optional build of fluent-bit.so
> +FLUENT_BIT_CONF_OPTS += -DFLB_SHARED_LIB=OFF -DBUILD_SHARED_LIBS=OFF
> +
> +# Don't build the examples
> +FLUENT_BIT_CONF_OPTS += -DFLB_EXAMPLES=OFF
> +
> +# onigmo option does not support cross compilation
> +FLUENT_BIT_CONF_OPTS += -DFLB_PARSER=OFF -DFLB_REGEX=OFF
> +
> +# backtrace option does not support cross compilation
> +FLUENT_BIT_CONF_OPTS += -DFLB_BACKTRACE=OFF
> +

Maybe it would be better to add a suboption to enable/disable
FLB_RECORD_ACCESSOR instead if requiring host-flex and host-bison being selected.

> +ifeq ($(BR2_PACKAGE_HOST_FLEX)$(BR2_PACKAGE_HOST_BISON),yy)
> +FLUENT_BIT_DEPENDENCIES += host-flex host-bison
> +FLUENT_BIT_CONF_OPTS += -DFLB_RECORD_ACCESSOR=ON -DFLB_STREAM_PROCESSOR=ON
> +else
> +FLUENT_BIT_CONF_OPTS += -DFLB_RECORD_ACCESSOR=OFF -DFLB_STREAM_PROCESSOR=OFF
> +endif
> +
> +ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),)
> +FLUENT_BIT_DEPENDENCIES += musl-fts
> +FLUENT_BIT_CONF_OPTS += -DCMAKE_EXE_LINKER_FLAGS=-lfts
> +endif
> +
> +# fts.h does not support _FILE_OFFSET_BITS=64 in glibc 2.22
> +FLUENT_BIT_CFLAGS = $(filter-out -D_FILE_OFFSET_BITS=64,$(TARGET_CFLAGS))
> +FLUENT_BIT_CXXFLAGS = $(filter-out -D_FILE_OFFSET_BITS=64,$(TARGET_CXXFLAGS))

Why do we want support old glibc ? It would disable LFS for glibc >= 2.23.

Best regards,
Romain

> +
> +FLUENT_BIT_CONF_OPTS += -DCMAKE_C_FLAGS="$(FLUENT_BIT_CFLAGS)" \
> +			-DCMAKE_CXX_FLAGS="$(FLUENT_BIT_CXXFLAGS)"
> +
> +define FLUENT_BIT_INSTALL_INIT_SYSTEMD
> +	$(INSTALL) -D -m 0644 $(@D)/init/fluent-bit.service \
> +		$(TARGET_DIR)/usr/lib/systemd/system/fluent-bit.service
> +	mkdir -p $(TARGET_DIR)/etc/systemd/system/multi-user.target.wants
> +	ln -sf ../../../..//usr/lib/systemd/system/fluent-bit.service \
> +		$(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/fluent-bit.service
> +endef
> +
> +# Install the default configuration files
> +define FLUENT_BIT_INSTALL_CONF
> +	mkdir -p $(TARGET_DIR)/etc/fluent-bit
> +	$(INSTALL) -m 0644 -D $(@D)/conf/fluent-bit.conf \
> +		$(TARGET_DIR)/etc/fluent-bit/fluent-bit.conf
> +	$(INSTALL) -m 0644 -D $(@D)/conf/plugins.conf \
> +		$(TARGET_DIR)/etc/fluent-bit/plugins.conf
> +endef
> +
> +FLUENT_BIT_POST_INSTALL_TARGET_HOOKS += FLUENT_BIT_INSTALL_CONF
> +
> +$(eval $(cmake-package))
>
Thomas Petazzoni Dec. 30, 2021, 2:53 p.m. UTC | #2 
Hello Frank,

On Mon, 17 Feb 2020 15:58:05 +0100
Frank Vanbever <frank.vanbever@essensium.com> wrote:

> From: Kevin Hilman <khilman@baylibre.com>
> 
> Fluent Bit is an open source and multi-platform Log Processor and
> Forwarder which allows you to collect data/logs from different
> sources, unify and send them to multiple destinations.
> 
> _FILE_OFFSET_BITS==64 needs to be filtered out to allow building with old glibc
> versions (<2.23).
> 
> 0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch is
> required to allow building for PowerPC.
> 
> Signed-off-by: Kevin Hilman <khilman@baylibre.com>
> Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>

I know it's our fault for not applying earlier, but the version 1.3.7
used in your package doesn't build with gcc 10.x (the infamous -fcommon
issue).

So I updated to 1.8.11, which also didn't build due to a "struct iovec"
issue. I backported upstream commit
b28d83d5851ca85943b5eea4f92b0c400169d4fb to fix this issue.

But then, it stills fails to build, because there are bunch of
warnings, and it is built with -Werror.

If you're still interested in this package, could you have a look?

Also Romain made some comments that were very true, and would need to
be handled as well.

Thanks a lot!

Thomas
Frank Vanbever Jan. 12, 2022, 4:59 p.m. UTC | #3 
Hi Thomas,

On Thursday, 30 December 2021 15:53:34 CET Thomas Petazzoni wrote:
> If you're still interested in this package, could you have a look?

I took another look and basically came to the same conclusion as you. In an 
ideal world I'd see it through, but my situation changed in the meantime and 
right now I don't have the bandwidth to dive into fixing those problems. Sorry 
about that.

Best regards,
Frank
diff mbox series

Patch

diff --git a/DEVELOPERS b/DEVELOPERS
index e4a3bcb24a..0c309d94e4 100644
--- a/DEVELOPERS
+++ b/DEVELOPERS
@@ -960,6 +960,7 @@  F:	package/zxing-cpp/
 
 N:	Frank Vanbever <frank.vanbever@essensium.com>
 F: 	package/elixir/
+F: 	package/fluent-bit/
 F:	package/libmodsecurity/
 F:	package/nginx-modsecurity/
 
diff --git a/package/Config.in b/package/Config.in
index dfa3f34b9d..f3d84f16e5 100644
--- a/package/Config.in
+++ b/package/Config.in
@@ -2298,6 +2298,7 @@  menu "System tools"
 	source "package/efibootmgr/Config.in"
 	source "package/efivar/Config.in"
 	source "package/emlog/Config.in"
+	source "package/fluent-bit/Config.in"
 	source "package/ftop/Config.in"
 	source "package/getent/Config.in"
 	source "package/htop/Config.in"
diff --git a/package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch b/package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch
new file mode 100644
index 0000000000..c85bf85d33
--- /dev/null
+++ b/package/fluent-bit/0001-lib-flb_libco-ppc-Fix-signature-for-co_create-API-18.patch
@@ -0,0 +1,42 @@ 
+From 731dab51a41d3635ceae5481d89d30148caac96d Mon Sep 17 00:00:00 2001
+From: Khem Raj <raj.khem@gmail.com>
+Date: Tue, 11 Feb 2020 19:36:21 -0800
+Subject: [PATCH] lib: flb_libco: ppc: Fix signature for co_create API (#1886)
+
+This patch fixes build errors for the flb_libco dependency when builidng
+for PowerPC.
+
+Backported from: 731dab51a41d3635ceae5481d89d30148caac96d
+Signed-off-by: Khem Raj <raj.khem@gmail.com>
+Signed-off-by: Frank Vanbever <frank.vanbever@essensium.com>
+---
+ lib/flb_libco/ppc.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/lib/flb_libco/ppc.c b/lib/flb_libco/ppc.c
+index e6536d56..533256b3 100644
+--- a/lib/flb_libco/ppc.c
++++ b/lib/flb_libco/ppc.c
+@@ -279,7 +279,9 @@ static uint32_t* co_create_(unsigned size, uintptr_t entry) {
+   return t;
+ }
+ 
+-cothread_t co_create(unsigned int size, void (*entry_)(void)) {
++cothread_t co_create(unsigned int size, void (*entry_)(void),
++                     size_t *out_size) {
++
+   uintptr_t entry = (uintptr_t)entry_;
+   uint32_t* t = 0;
+ 
+@@ -325,7 +327,7 @@ cothread_t co_create(unsigned int size, void (*entry_)(void)) {
+     t[10] = (uint32_t)(sp >> shift >> shift);
+     t[11] = (uint32_t)sp;
+   }
+-
++  *out_size = size;
+   return t;
+ }
+ 
+-- 
+2.20.1
+
diff --git a/package/fluent-bit/Config.in b/package/fluent-bit/Config.in
new file mode 100644
index 0000000000..98efaa36ff
--- /dev/null
+++ b/package/fluent-bit/Config.in
@@ -0,0 +1,15 @@ 
+config BR2_PACKAGE_FLUENT_BIT
+	bool "fluent-bit"
+	depends on BR2_INSTALL_LIBSTDCPP
+	depends on !BR2_STATIC_LIBS
+	select BR2_PACKAGE_MUSL_FTS if !BR2_TOOLCHAIN_USES_GLIBC
+	help
+	  Fluent Bit is an open source and multi-platform Log
+	  processor and forwarder which allows you to collect
+	  data/logs from different sources, unify and send them to
+	  multiple destinations.
+
+	  http://fluentbit.io
+
+comment "fluent-bit needs a toolchain w/ C++, dynamic library"
+	depends on !BR2_INSTALL_LIBSTDCPP || BR2_STATIC_LIBS
diff --git a/package/fluent-bit/fluent-bit.hash b/package/fluent-bit/fluent-bit.hash
new file mode 100644
index 0000000000..1a8e0d121c
--- /dev/null
+++ b/package/fluent-bit/fluent-bit.hash
@@ -0,0 +1,3 @@ 
+# computed locally
+sha256  e897949356855e2a7d1a71085c7c21653b7792ef137d19d665e7b5dcd8e4d46a  fluent-bit-1.3.7.tar.gz
+sha256  0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594  LICENSE
diff --git a/package/fluent-bit/fluent-bit.mk b/package/fluent-bit/fluent-bit.mk
new file mode 100644
index 0000000000..c48d77a1fe
--- /dev/null
+++ b/package/fluent-bit/fluent-bit.mk
@@ -0,0 +1,65 @@ 
+################################################################################
+#
+# fluent-bit
+#
+################################################################################
+
+FLUENT_BIT_VERSION = 1.3.7
+FLUENT_BIT_SITE = $(call github,fluent,fluent-bit,v$(FLUENT_BIT_VERSION))
+FLUENT_BIT_LICENSE = Apache-2.0
+FLUENT_BIT_LICENSE_FILE = LICENSE
+
+# LuaJIT and filter_lua do not support cross compilation
+FLUENT_BIT_CONF_OPTS += -DFLB_LUAJIT=OFF -DFLB_FILTER_LUA=OFF
+
+# Only build the daemon. Disable optional build of fluent-bit.so
+FLUENT_BIT_CONF_OPTS += -DFLB_SHARED_LIB=OFF -DBUILD_SHARED_LIBS=OFF
+
+# Don't build the examples
+FLUENT_BIT_CONF_OPTS += -DFLB_EXAMPLES=OFF
+
+# onigmo option does not support cross compilation
+FLUENT_BIT_CONF_OPTS += -DFLB_PARSER=OFF -DFLB_REGEX=OFF
+
+# backtrace option does not support cross compilation
+FLUENT_BIT_CONF_OPTS += -DFLB_BACKTRACE=OFF
+
+ifeq ($(BR2_PACKAGE_HOST_FLEX)$(BR2_PACKAGE_HOST_BISON),yy)
+FLUENT_BIT_DEPENDENCIES += host-flex host-bison
+FLUENT_BIT_CONF_OPTS += -DFLB_RECORD_ACCESSOR=ON -DFLB_STREAM_PROCESSOR=ON
+else
+FLUENT_BIT_CONF_OPTS += -DFLB_RECORD_ACCESSOR=OFF -DFLB_STREAM_PROCESSOR=OFF
+endif
+
+ifeq ($(BR2_TOOLCHAIN_USES_GLIBC),)
+FLUENT_BIT_DEPENDENCIES += musl-fts
+FLUENT_BIT_CONF_OPTS += -DCMAKE_EXE_LINKER_FLAGS=-lfts
+endif
+
+# fts.h does not support _FILE_OFFSET_BITS=64 in glibc 2.22
+FLUENT_BIT_CFLAGS = $(filter-out -D_FILE_OFFSET_BITS=64,$(TARGET_CFLAGS))
+FLUENT_BIT_CXXFLAGS = $(filter-out -D_FILE_OFFSET_BITS=64,$(TARGET_CXXFLAGS))
+
+FLUENT_BIT_CONF_OPTS += -DCMAKE_C_FLAGS="$(FLUENT_BIT_CFLAGS)" \
+			-DCMAKE_CXX_FLAGS="$(FLUENT_BIT_CXXFLAGS)"
+
+define FLUENT_BIT_INSTALL_INIT_SYSTEMD
+	$(INSTALL) -D -m 0644 $(@D)/init/fluent-bit.service \
+		$(TARGET_DIR)/usr/lib/systemd/system/fluent-bit.service
+	mkdir -p $(TARGET_DIR)/etc/systemd/system/multi-user.target.wants
+	ln -sf ../../../..//usr/lib/systemd/system/fluent-bit.service \
+		$(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/fluent-bit.service
+endef
+
+# Install the default configuration files
+define FLUENT_BIT_INSTALL_CONF
+	mkdir -p $(TARGET_DIR)/etc/fluent-bit
+	$(INSTALL) -m 0644 -D $(@D)/conf/fluent-bit.conf \
+		$(TARGET_DIR)/etc/fluent-bit/fluent-bit.conf
+	$(INSTALL) -m 0644 -D $(@D)/conf/plugins.conf \
+		$(TARGET_DIR)/etc/fluent-bit/plugins.conf
+endef
+
+FLUENT_BIT_POST_INSTALL_TARGET_HOOKS += FLUENT_BIT_INSTALL_CONF
+
+$(eval $(cmake-package))