Please add package varnish to OpenWRT

BigG · November 21, 2023, 3:57am

Hello,
I would like to use varnish on OpenWRT for x86.
Thank you

andyboeh · November 21, 2023, 9:03am

The way to push this forward is simple: Port it to OpenWrt yourself and create a PR / send it to the mailing list to include it upstream.

As a starting point, a quick web search provided an old port: https://github.com/heil/owrt-package/tree/master/net/varnish

BigG · November 21, 2023, 11:29pm

If it would be that simply I would have done it, I found that via google already but it doesn't work with a current version of varnish even if you update the makefile

PKG_NAME:=varnish
PKG_VERSION:=7.4.2
PKG_RELEASE:=1

PKG_SOURCE:=varnish-$(PKG_VERSION).tar.gz
PKG_SOURCE_URL:=https://varnish-cache.org/downloads/
PKG_HASH:=6d3d03c67514e6bb4e8584e40a381f51e708607d39337a63dc4ae42061d9a46f

So adding support is something for somebody who does actually know what they do when it comes to creating packages for OpenWRT.

andyboeh · November 22, 2023, 7:34pm

People here can assist you in porting, but stating "it doesn't work" doesn't provide any details. What have you tried? What error messages do you get?

BigG · November 22, 2023, 10:16pm

makefile

#
# Copyright (C) 2010 OpenWrt.org
#
# This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
#

include $(TOPDIR)/rules.mk

PKG_NAME:=varnish
PKG_VERSION:=7.4.2
PKG_RELEASE:=1

PKG_SOURCE:=varnish-$(PKG_VERSION).tgz
PKG_SOURCE_URL:=https://varnish-cache.org/downloads/
PKG_HASH:=6d3d03c67514e6bb4e8584e40a381f51e708607d39337a63dc4ae42061d9a46f

#PKG_FIXUP:=autoreconf

include $(INCLUDE_DIR)/package.mk

define Package/varnish
  SUBMENU:=Web Servers/Proxies
  SECTION:=net
  CATEGORY:=Network
  TITLE:=VARNISH CACHE make Websites fly
  URL:=http://www.varnish-cache.org/
  DEPENDS:=+libpcre +libltdl +libncurses +logrotate +librt +bash +zlib  +libelf +libreadline +libncursesw +libunwind
  USERID:=varnish=199:varnish=199
endef

define Package/varnish/description
  Open source Website Cache
endef

CONFIGURE_VARS += \
	ac_cv_so_rcvtimeo_works=no \
	ac_cv_so_sndtimeo_works=no \
	ac_cv_so_sendfile_works=no \
	ac_cv_lib_jemalloc_malloc_conf=no \
	ac_cv_have_tcp_keep=yes \
	ac_cv_have_viz=no \
	VCC_CC="x86_64-openwrt-linux-gnu-gcc -std=gnu99 -O2 -march=x86-64 -pipe -fomit-frame-pointer -fno-caller-saves  -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -Wformat -Werror=format-security -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -Wall -Werror -Wno-error=unused-result -pthread -fpic -shared -Wl,-x -o %o %s"
	#VCC_CC="varnishcc %s %o" 

define Build/Configure
        (cd $(PKG_BUILD_DIR); rm -f config.cache; \
		VCC_CC="x86_64-openwrt-linux-gnu-gcc -std=gnu99 -O2 -march=x86-64 -pipe -fomit-frame-pointer -fno-caller-saves  -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -Wformat -Werror=format-security -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -Wall -Werror -Wno-error=unused-result -pthread -fpic -shared -Wl,-x -o %o %s" \
                $(TARGET_CONFIGURE_OPTS) \
                CFLAGS="$(TARGET_CFLAGS)" \
                CPPFLAGS="$(TARGET_CPPFLAGS)" \
                LDFLAGS="$(TARGET_LDFLAGS)" \
                ./configure \
                --host=$(REAL_GNU_TARGET_NAME) \
                --target=$(REAL_GNU_TARGET_NAME) \
                -disable-nls \
                --prefix=/usr \
                --exec-prefix=/usr \
                --disable-pcre-jit \
                --program-transform-name="" \
        );
endef

define Build/Compile
	$(MAKE) -j4 -C $(PKG_BUILD_DIR) \
		CC="$(TARGET_CC)" \
		CFLAGS="$(TARGET_CFLAGS) -std=gnu99" \
		LD="$(TARGET_CC)" \
		LDFLAGS="$(TARGET_LDFLAGS) -lpcre -lpthread" \
		DESTDIR="$(PKG_INSTALL_DIR)" \
	install
	
	mkdir -p $(PKG_INSTALL_DIR)/etc
	$(CP) $(PKG_BUILD_DIR)/etc/builtin.vcl $(PKG_INSTALL_DIR)/etc

	$(MAKE) -C $(PKG_BUILD_DIR)/bin/varnishtop \
		CC="$(TARGET_CC)" \
		CFLAGS="$(TARGET_CFLAGS)" \
		LD="$(TARGET_CC)" \
		LDFLAGS="$(TARGET_LDFLAGS) -lpcre -lpthread" \
		DESTDIR="$(PKG_INSTALL_DIR)" \
	install
endef

define Package/varnish/conffiles
/etc/varnish/default.vcl
/etc/config/varnish
endef

define Package/varnish/install
	$(INSTALL_DIR) $(1)/etc/varnish
	$(INSTALL_CONF) $(PKG_INSTALL_DIR)/etc/builtin.vcl $(1)/etc/varnish
	$(INSTALL_CONF) ./files/config.vcl $(1)/etc/varnish
	chmod 0754 $(1)/etc/varnish/config.vcl
	$(INSTALL_DIR) $(1)/etc/init.d
	$(INSTALL_CONF) ./files/varnish.init $(1)/etc/init.d/varnish
	chmod 0750 $(1)/etc/init.d/varnish
	$(INSTALL_DIR) $(1)/etc/config
	$(INSTALL_CONF) ./files/varnish.config $(1)/etc/config/varnish
	$(INSTALL_DIR) $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishadm $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishhist $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishlog $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishncsa $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishstat $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishtest $(1)/usr/bin
	$(INSTALL_DIR) $(1)/usr/sbin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/sbin/varnishd $(1)/usr/sbin/
	$(INSTALL_DIR) $(1)/etc/varnish
	$(INSTALL_DIR) $(1)/usr/lib

	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/lib/*.so* $(1)/usr/lib/
	$(INSTALL_DIR) $(1)/usr/lib/varnish
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/lib/varnish/*.so* $(1)/usr/lib/varnish
	$(INSTALL_DIR) $(1)/usr/lib/varnish/vmods
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/lib/varnish/vmods/* $(1)/usr/lib/varnish/vmods
	$(INSTALL_BIN) ./files/pts-tcc $(1)/usr/bin
	chmod 0750 $(1)/usr/bin/pts-tcc
	$(INSTALL_BIN) ./files/varnishcc $(1)/usr/bin/varnishcc
	chmod 0750 $(1)/usr/bin/varnishcc
endef

define Package/varnish-top
	MENU:=1
	$(call Package/varnish)
	TITLE+= varnish-top)
	DEPENDS:=varnish
endef

define Package/varnish-top/description
	varnishtop top for varnish
endef

define Package/varnish-top/install
	$(INSTALL_DIR) $(1)/usr/bin
	$(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/bin/varnishtop $(1)/usr/bin/
endef

$(eval $(call BuildPackage,varnish))
$(eval $(call BuildPackage,varnish-top))

error

checking for TCP_KEEP(CNT|IDLE|INTVL) socket options... configure: error: in `/home/user/openwrt/build_dir/target-x86_64_musl/varnish-7.4.2':
configure: error: cannot run test program while cross compiling
See `config.log' for more details
make[2]: *** [Makefile:143: /home/user/openwrt/build_dir/target-x86_64_musl/varnish-7.4.2/.configured_68b329da9893e34099c7d8ad5cb9c940] Error 1
make[2]: Leaving directory '/home/user/openwrt/feeds/packages/net/varnish'
time: package/feeds/packages/varnish/compile#1.84#0.47#2.13
    ERROR: package/feeds/packages/varnish failed to build.
make[1]: *** [package/Makefile:128: package/feeds/packages/varnish/compile] Error 1
make[1]: Leaving directory '/home/user/openwrt'
make: *** [/home/user/openwrt/include/toplevel.mk:232: package/varnish/compile] Error 2

config.log does not get created so I can't post it.

stangri · November 22, 2023, 11:21pm

FWIW: based on the Makefile, I realize you made it for your personal needs and don't plan to send PR for it to the packages repo, however, AFAIK, pcre is on the way out and being actively replaced by pcre2 in the tree. So you may be able to build it for OpenWrt now, but won't be in the near future.

andyboeh · November 23, 2023, 4:34am

Usually, there is a configure option to disable running tests. This needs to be added to the Configure step so that it succeeds. Check out the options to the configure script, e.g. by running it manually with --help.

BigG · November 23, 2023, 7:07pm

I have no clue about what you write there

andyboeh · November 23, 2023, 7:50pm

On Linux, packages are often built with the commands:

./configure
make
make install

This configure-script is run by the OpenWrt build system, and this is what fails. I just checked for you, you need to override configure's assumption about your system. This question on stackexchange basically explains this.

The part

CONFIGURE_VARS += \
	ac_cv_so_rcvtimeo_works=no \
	ac_cv_so_sndtimeo_works=no \
	ac_cv_so_sendfile_works=no \
	ac_cv_lib_jemalloc_malloc_conf=no \
	ac_cv_have_tcp_keep=yes \
	ac_cv_have_viz=no \
	VCC_CC="x86_64-openwrt-linux-gnu-gcc -std=gnu99 -O2 -march=x86-64 -pipe -fomit-frame-pointer -fno-caller-saves  -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -Wformat -Werror=format-security -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -Wall -Werror -Wno-error=unused-result -pthread -fpic -shared -Wl,-x -o %o %s"
	#VCC_CC="varnishcc %s %o"

overrides the configure variables, so you need to identify what configure checks/expects when it fails and add the correct answer to this section of the Makefile.

Spoiler - to check your result

I checked the configure script for you. The variable to set is ac_cv_have_tcp_keep and I assume (NB: I did not test it) it needs to be yes. Thus, CONFIGURE_VARS now reads:

CONFIGURE_VARS += \
	ac_cv_so_rcvtimeo_works=no \
	ac_cv_so_sndtimeo_works=no \
	ac_cv_so_sendfile_works=no \
	ac_cv_lib_jemalloc_malloc_conf=no \
	ac_cv_have_tcp_keep=yes \
	ac_cv_have_viz=no \
	ac_cv_have_tcp_keep=yes \
	VCC_CC="x86_64-openwrt-linux-gnu-gcc -std=gnu99 -O2 -march=x86-64 -pipe -fomit-frame-pointer -fno-caller-saves  -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -Wformat -Werror=format-security -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -Wall -Werror -Wno-error=unused-result -pthread -fpic -shared -Wl,-x -o %o %s"
	#VCC_CC="varnishcc %s %o"

Then on to the next error or...?

BigG · November 23, 2023, 9:54pm

After changing it to your vars it still errors out with the same error

checking for TCP_KEEP(CNT|IDLE|INTVL) socket options... configure: error: in `/home/user/openwrt/build_dir/target-x86_64_musl/varnish-7.4.2':
configure: error: cannot run test program while cross compiling
See `config.log' for more details
make[2]: *** [Makefile:144: /home/user/openwrt/build_dir/target-x86_64_musl/varnish-7.4.2/.configured_68b329da9893e34099c7d8ad5cb9c940] Error 1
make[2]: Leaving directory '/home/user/openwrt/feeds/packages/net/varnish'
time: package/feeds/packages/varnish/compile#1.84#0.35#2.21
    ERROR: package/feeds/packages/varnish failed to build.
make[1]: *** [package/Makefile:128: package/feeds/packages/varnish/compile] Error 1
make[1]: Leaving directory '/home/user/openwrt'
make: *** [/home/user/openwrt/include/toplevel.mk:232: package/varnish/compile] Error 2

stangri · November 24, 2023, 2:16am

@andyboeh can you please explain how to find out which variables need to be set up manually in general and how to troubleshoot the ./configure errors (or are they self-evident)?

andyboeh · November 24, 2023, 6:06am

You might have to reinstall the package feed and/or run make package/varnish/clean first. Unfortunately, I don't have a stable internet connection atm, but I might be able to check on the weekend.

I have zero experience with this as well, but I just noticed that this is also explained in the Wiki: https://openwrt.org/docs/guide-developer/packages.flags

I searched for the error message in configure and went through the code to find the variable.

BigG · November 24, 2023, 10:56am

I did that and even did a targetclean but still same error

predators_46 · December 19, 2023, 2:04am

hy @stangri

root@OpenWrt:/usr/bin# ldd varnishadm
        /lib/ld-musl-aarch64.so.1 (0x7fa7e0a000)
        libvarnishapi.so.3 => /usr/lib/libvarnishapi.so.3 (0x7fa7db9000)
        libpcre2-8.so.0 => /usr/lib/libpcre2-8.so.0 (0x7fa7d58000)
        libedit.so.0 => /usr/lib/libedit.so.0 (0x7fa7d14000)
        libgcc_s.so.1 => /lib/libgcc_s.so.1 (0x7fa7ce3000)
        libc.so => /lib/ld-musl-aarch64.so.1 (0x7fa7e0a000)
        libncursesw.so.6 => /usr/lib/libncursesw.so.6 (0x7fa7c72000)