[FIXED - used flup] Error compiling uwsgi with Python3 (LONG_INT)

For Developers
1

Hi all!
I've decided to go with python3 for my latest project and I'm failing to build uwsgi.
I'm building on 17.0.1 branch (I confess I didn't try on master yet).
I'm currently building on a 64bit Linux Mint 18 and my target is a ramips/mt7620.

For reference, it works well with python2 on the same build environment (and same version of uwsgi).

My Makefile is as follow:

include $(TOPDIR)/rules.mk

PKG_NAME:=python3-uwsgi
PKG_VERSION:=2.0.17
PKG_RELEASE:=1
PKG_LICENSE:=GPL2

PKG_SOURCE:=uwsgi-$(PKG_VERSION).tar.gz
PKG_HASH:=3dc2e9b48db92b67bfec1badec0d3fdcc0771316486c5efa3217569da3528bf2
PKG_SOURCE_URL:=https://pypi.python.org/packages/98/b2/19b34b20662d111f7d2f926cdf10e13381761dd7dbd10666b9076cbdcd22/
PKG_BUILD_DIR:=$(BUILD_DIR)/uwsgi-$(PKG_VERSION)

include $(INCLUDE_DIR)/package.mk
$(call include_mk, python3-package.mk)

define Package/$(PKG_NAME)
	SECTION:=language-python
	CATEGORY:=Languages
	SUBMENU:=Python
	TITLE:=The uWSGI server
	URL:=http://unbit.com/
	DEPENDS:=+python3 +python3-setuptools +libopenssl +libpcre +jansson +libcap +libuuid
endef

define Package/$(PKG_NAME)/description
	The uWSGI project
endef

define Build/Compile
	$(call Build/Compile/Py3Mod,,install --prefix=/usr --root="$(PKG_INSTALL_DIR)")
endef

define Package/$(PKG_NAME)/install
	$(INSTALL_DIR) $(1)$(PYTHON3_PKG_DIR)
	$(INSTALL_DIR) $(1)/usr/bin
	$(CP) \
		$(PKG_INSTALL_DIR)$(PYTHON3_PKG_DIR)/* \
		$(1)$(PYTHON3_PKG_DIR)/
	$(CP) \
		$(PKG_INSTALL_DIR)/usr/bin/* \
		$(1)/usr/bin/
endef

$(eval $(call BuildPackage,$(PKG_NAME)))

and the errors I get during the compilation are around LONG_BIT conflicting definition:

make[2]: Entering directory '/home/x/repos/lede/feeds/ui/feed/python3-uwsgi'
. /home/x/repos/lede/include/shell.sh; gzip -dc /home/x/.openwrt_dl/uwsgi-2.0.17.tar.gz | tar -C /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/.. -xf -
[ ! -d ./src/ ] || cp -fpR ./src/* /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17
touch /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/.prepared_8a0a853c636dab9e8c87f17f4dd3822568b329da9893e34099c7d8ad5cb9c940
rm -f /home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/stamp/.python3-uwsgi_installed
(cd /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/./; if [ -x ./configure ]; then find /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ -name config.guess | xargs -r chmod u+w; find /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ -name config.guess | xargs -r -n1 cp --remove-destination /home/x/repos/lede/scripts/config.guess; find /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ -name config.sub | xargs -r chmod u+w; find /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ -name config.sub | xargs -r -n1 cp --remove-destination /home/x/repos/lede/scripts/config.sub; AR="mipsel-openwrt-linux-musl-gcc-ar" AS="mipsel-openwrt-linux-musl-gcc -c -O3 -pipe -mno-branch-likely -mips32r2 -mtune=24kc -mdsp -fno-caller-saves -fno-plt -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -msoft-float -iremap/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17:uwsgi-2.0.17 -Wformat -Werror=format-security -fstack-protector -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -mno-mips16 -mno-interlink-mips16" LD=mipsel-openwrt-linux-musl-ld NM="mipsel-openwrt-linux-musl-gcc-nm" CC="mipsel-openwrt-linux-musl-gcc" GCC="mipsel-openwrt-linux-musl-gcc" CXX="mipsel-openwrt-linux-musl-g++" RANLIB="mipsel-openwrt-linux-musl-gcc-ranlib" STRIP=mipsel-openwrt-linux-musl-strip OBJCOPY=mipsel-openwrt-linux-musl-objcopy OBJDUMP=mipsel-openwrt-linux-musl-objdump SIZE=mipsel-openwrt-linux-musl-size CFLAGS="-O3 -pipe -mno-branch-likely -mips32r2 -mtune=24kc -mdsp -fno-caller-saves -fno-plt -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -msoft-float -mips16 -minterlink-mips16 -iremap/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17:uwsgi-2.0.17 -Wformat -Werror=format-security -fstack-protector -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -mno-mips16 -mno-interlink-mips16 " CXXFLAGS="-O3 -pipe -mno-branch-likely -mips32r2 -mtune=24kc -mdsp -fno-caller-saves -fno-plt -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -msoft-float -mips16 -minterlink-mips16 -iremap/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17:uwsgi-2.0.17 -Wformat -Werror=format-security -fstack-protector -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -mno-mips16 -mno-interlink-mips16 " CPPFLAGS="-I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/include -I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/include -I/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/usr/include -I/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/include/fortify -I/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/include " LDFLAGS="-L/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/lib -L/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/lib -L/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/usr/lib -L/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/lib -znow -zrelro "   ./configure --target=mipsel-openwrt-linux --host=mipsel-openwrt-linux --build=x86_64-linux-gnu --program-prefix="" --program-suffix="" --prefix=/usr --exec-prefix=/usr --bindir=/usr/bin --sbindir=/usr/sbin --libexecdir=/usr/lib --sysconfdir=/etc --datadir=/usr/share --localstatedir=/var --mandir=/usr/man --infodir=/usr/info --disable-nls --disable-ipv6 _python_sysroot="/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16" _python_prefix="/usr" _python_exec_prefix="/usr" ; fi; )
rm -f /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/.configured_*
touch /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/.configured_01d0ed13b2aae96bdf3b133bc55801d7
install -d -m0755 /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ipkg-install//usr/lib/python3.6/site-packages
if [ "" == "HOST" ]; then export PYTHONPATH="/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/host/lib/python3.6:/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/host/lib/python3.6/site-packages"; export PYTHONDONTWRITEBYTECODE=0; else export PYTHONPATH="/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/lib/python3.6:/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16//usr/lib/python3.6/site-packages:/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ipkg-install//usr/lib/python3.6/site-packages"; export PYTHONDONTWRITEBYTECODE=1; export _python_sysroot="/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16"; export _python_prefix="/usr"; export _python_exec_prefix="/usr"; fi; export PYTHONOPTIMIZE="";   cd /home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/; CC="mipsel-openwrt-linux-musl-gcc" CCSHARED="mipsel-openwrt-linux-musl-gcc -fpic" CXX="mipsel-openwrt-linux-musl-g++" LD="mipsel-openwrt-linux-musl-gcc" LDSHARED="mipsel-openwrt-linux-musl-gcc -shared" CFLAGS="-O3 -pipe -mno-branch-likely -mips32r2 -mtune=24kc -mdsp -fno-caller-saves -fno-plt -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -msoft-float -mips16 -minterlink-mips16 -iremap/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17:uwsgi-2.0.17 -Wformat -Werror=format-security -fstack-protector -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -mno-mips16 -mno-interlink-mips16" CPPFLAGS="-I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/include -I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/include -I/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/usr/include -I/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/include/fortify -I/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/include -I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/include/python3.6" LDFLAGS="-L/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/lib -L/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/lib -L/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/usr/lib -L/home/x/repos/lede/staging_dir/toolchain-mipsel_24kc_gcc-5.4.0_musl-1.1.16/lib -znow -zrelro -lpython3.6" _PYTHON_HOST_PLATFORM=linux2 __PYVENV_LAUNCHER__="/usr/bin/python3.6"    /home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/host/bin/python3.6   ./setup.py install --prefix=/usr --root="/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/ipkg-install" ;
/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/lib/python3.6/distutils/dist.py:261: UserWarning: Unknown distribution option: 'descriptions'
  warnings.warn(msg)
running install
using profile: buildconf/default.ini
detected include path: ['/usr/lib/gcc/x86_64-linux-gnu/5/include', '/usr/local/include', '/usr/lib/gcc/x86_64-linux-gnu/5/include-fixed', '/usr/include/x86_64-linux-gnu', '/usr/include']
Patching "bin_name" to properly install_scripts dir
detected CPU cores: 4
configured CFLAGS: -O2 -I. -Wall -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -pipe -mno-branch-likely -mips32r2 -mtune=24kc -mdsp -fno-caller-saves -fno-plt -fhonour-copts -Wno-error=unused-but-set-variable -Wno-error=unused-result -msoft-float -mips16 -minterlink-mips16 -iremap/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17:uwsgi-2.0.17 -Wformat -Werror=format-security -fstack-protector -D_FORTIFY_SOURCE=1 -Wl,-z,now -Wl,-z,relro -mno-mips16 -mno-interlink-mips16 -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -DUWSGI_HAS_IFADDRS -DUWSGI_ZLIB -DUWSGI_LOCK_USE_MUTEX -DUWSGI_EVENT_USE_EPOLL -DUWSGI_EVENT_TIMER_USE_TIMERFD -DUWSGI_EVENT_FILEMONITOR_USE_INOTIFY  -DUWSGI_PCRE -DUWSGI_ROUTING -DUWSGI_VERSION="\"2.0.17\"" -DUWSGI_VERSION_BASE="2" -DUWSGI_VERSION_MAJOR="0" -DUWSGI_VERSION_MINOR="17" -DUWSGI_VERSION_REVISION="0" -DUWSGI_VERSION_CUSTOM="\"\"" -DUWSGI_YAML -I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/include -DUWSGI_JSON -DUWSGI_SSL -I/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/usr/include/libxml2 -DUWSGI_XML -DUWSGI_XML_LIBXML2 -DUWSGI_PLUGIN_DIR="\".\"" -DUWSGI_DECLARE_EMBEDDED_PLUGINS="UDEP(python);UDEP(gevent);UDEP(ping);UDEP(cache);UDEP(nagios);UDEP(rrdtool);UDEP(carbon);UDEP(rpc);UDEP(corerouter);UDEP(fastrouter);UDEP(http);UDEP(signal);UDEP(syslog);UDEP(rsyslog);UDEP(logsocket);UDEP(router_uwsgi);UDEP(router_redirect);UDEP(router_basicauth);UDEP(zergpool);UDEP(redislog);UDEP(mongodblog);UDEP(router_rewrite);UDEP(router_http);UDEP(logfile);UDEP(router_cache);UDEP(rawrouter);UDEP(router_static);UDEP(sslrouter);UDEP(spooler);UDEP(cheaper_busyness);UDEP(symcall);UDEP(transformation_tofile);UDEP(transformation_gzip);UDEP(transformation_chunked);UDEP(transformation_offload);UDEP(router_memcached);UDEP(router_redis);UDEP(router_hash);UDEP(router_expires);UDEP(router_metrics);UDEP(transformation_template);UDEP(stats_pusher_socket);" -DUWSGI_LOAD_EMBEDDED_PLUGINS="ULEP(python);ULEP(gevent);ULEP(ping);ULEP(cache);ULEP(nagios);ULEP(rrdtool);ULEP(carbon);ULEP(rpc);ULEP(corerouter);ULEP(fastrouter);ULEP(http);ULEP(signal);ULEP(syslog);ULEP(rsyslog);ULEP(logsocket);ULEP(router_uwsgi);ULEP(router_redirect);ULEP(router_basicauth);ULEP(zergpool);ULEP(redislog);ULEP(mongodblog);ULEP(router_rewrite);ULEP(router_http);ULEP(logfile);ULEP(router_cache);ULEP(rawrouter);ULEP(router_static);ULEP(sslrouter);ULEP(spooler);ULEP(cheaper_busyness);ULEP(symcall);ULEP(transformation_tofile);ULEP(transformation_gzip);ULEP(transformation_chunked);ULEP(transformation_offload);ULEP(router_memcached);ULEP(router_redis);ULEP(router_hash);ULEP(router_expires);ULEP(router_metrics);ULEP(transformation_template);ULEP(stats_pusher_socket);"
*** uWSGI compiling server core ***
[thread 1][mipsel-openwrt-linux-musl-gcc] core/utils.o
[thread 2][mipsel-openwrt-linux-musl-gcc] core/protocol.o

... All good here - removing ...

[thread 3][mipsel-openwrt-linux-musl-gcc] core/dot_h.o
[thread 3][mipsel-openwrt-linux-musl-gcc] core/config_py.o
*** uWSGI compiling embedded plugins ***
[thread 3][mipsel-openwrt-linux-musl-gcc] plugins/python/python_plugin.o
[thread 0][mipsel-openwrt-linux-musl-gcc] plugins/python/pyutils.o
In file included from /home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/host/include/python3.6/Python.h:50:0,
                 from plugins/python/uwsgi_python.h:2,
                 from plugins/python/python_plugin.c:1:
/home/x/repos/lede/staging_dir/target-mipsel_24kc_musl-1.1.16/host/include/python3.6/pyport.h:686:2: error: #error "LONG_BIT definition appears wrong for platform (bad gcc/glibc config?)."
 #error "LONG_BIT definition appears wrong for platform (bad gcc/glibc config?)."
  ^
Makefile:43: recipe for target '/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/.built' failed
make[2]: *** [/home/x/repos/lede/build_dir/target-mipsel_24kc_musl-1.1.16/uwsgi-2.0.17/.built] Error 1
make[2]: Leaving directory '/home/x/repos/lede/feeds/ui/feed/python3-uwsgi'
package/Makefile:105: recipe for target 'package/feeds/ui/python3-uwsgi/compile' failed
make[1]: *** [package/feeds/ui/python3-uwsgi/compile] Error 2
make[1]: Leaving directory '/home/x/repos/lede'
/home/x/repos/lede/include/toplevel.mk:205: recipe for target 'package/python3-uwsgi/compile' failed
make: *** [package/python3-uwsgi/compile] Error 2

Any ideas as to how to deal with this?

P.S. credits and thanks to this post which got me started and also made me try (and succeed) with python 2

2

That kind of error is unfortunately common when trying to compile on "non-standard" platforms. Between endian-ness and word length, the C-type definitions can be "off".

There are a few places that it comes from, in my experience:

  • GNU "config" tools that fail on non-standard platforms
  • GNU config tools that fail in cross-compile situations
  • Lack of inclusion of proper headers, or inclusion in the "wrong" order for the platform
  • "Bad" definitions in <types.h> or <sys/types.h> or related headers

Adding to that list,

detected include path: ['/usr/lib/gcc/x86_64-linux-gnu/5/include', '/usr/local/include', '/usr/lib/gcc/x86_64-linux-gnu/5/include-fixed', '/usr/include/x86_64-linux-gnu', '/usr/include']

suggests that the Python build process may be picking up the host definitions, not the target definitions.

uwsgi-2.0.17$ fgrep -Ir "detected include path" .
./uwsgiconfig.py:            print("detected include path: %s" % self.include_path)

There may be some hints as to how to modify your Makefile in something like feeds/packages/lang/python/python-cffi/Makefile

1 Like
3

thanks @jeff,
I suspected something like that and had realised that these includes were the host ones.
Since the issue was in what looked like OpenWRT paths, I was hoping that the above issue was cosmetic.

I now have a feeling that if the python 2 version builds could be a fluke maybe due to the fact that the same version of python happens to be the default one on my system.

I'll keep digging, thanks!

4

For reference, after digging deep, I managed to build the core and basic package (option nolang) but would still hit the issue when trying the option pyonly. If you need non python plugins, you may be able to do so by using the correct target (ini files in the buildconf directory of uwsgi).

something like:

define Build/Compile
	$(INSTALL_DIR) $(PKG_INSTALL_DIR)/$(PYTHON3_PKG_DIR)
	$(call Build/Compile/HostPy3RunTarget, \
		cd $(PKG_BUILD_DIR), \
		uwsgiconfig.py --build nolang, \
		)
endef

I had managed to deal with these host includes by setting the environment variable CPP with no visible change.
I also played with the UWSGI_INCLUDES to see if I could so something but wasn't successful.

For reference, I went with flup (1.0.3) which built fine and will do the trick when it comes to FastCGI.
I'm considering this solved on my part but haven't actually built uwsgi with Python 3.

5

Would you mind sharing the Makefile for flup with python3 changes? I replaced python with python3 and pymod with py3mod, still some error. Wondering what could be the problem.

TiA
zer0_0ne

6

@zer0_0ne,
no probs, there's a couple of places where an additional '3' is needed.
Here's my makefile. I hope that helps!

Makefile 
include $(TOPDIR)/rules.mk

PKG_NAME:=python3-flup
PKG_VERSION:=1.0.3
PKG_RELEASE:=1
PKG_LICENSE:=BSD-3-Clause

PKG_SOURCE:=flup-$(PKG_VERSION).tar.gz
PKG_SOURCE_URL:=http://pypi.python.org/packages/source/f/flup/
PKG_MD5SUM:=a64e7a6374e043480ee92534c735964e
PKG_BUILD_DEPENDS:=python3 python3-setuptools
PKG_BUILD_DIR:=$(BUILD_DIR)/flup-$(PKG_VERSION)

include $(INCLUDE_DIR)/package.mk
$(call include_mk, python3-package.mk)

define Package/$(PKG_NAME)
	SUBMENU:=Python
	SECTION:=lang
	CATEGORY:=Languages
	TITLE:=Random assortment of WSGI servers
	URL:=http://www.saddi.com/software/flup/
	DEPENDS:=+python3
endef

define Package/$(PKG_NAME)/description
  Random assortment of WSGI servers
endef

define Build/Compile
	$(call Build/Compile/Py3Mod,,install --prefix=/usr --root=$(PKG_INSTALL_DIR))
endef

define Package/$(PKG_NAME)/install
	$(INSTALL_DIR) $(1)$(PYTHON3_PKG_DIR)
	$(CP) \
	    $(PKG_INSTALL_DIR)$(PYTHON3_PKG_DIR)/* \
	    $(1)$(PYTHON3_PKG_DIR)
endef

$(eval $(call BuildPackage,$(PKG_NAME)))
7

Thanks Vincent,

Thanks for sharing the Makefile. I figured it out too. Turns out there are two versions of flup one for 2.7 and other python 3.x. I was struggling with 2.x on Python 3.x. I switched to flup-py3 and it worked as expected.

zer0_0ne

1 Like
8

You're right, I seem to remember that I had to use the latest flup which supports Python 3. I wasn't convinced by the other implementations on github as they don't seem to be maintained much.

9

This topic was automatically closed 6 days after the last reply. New replies are no longer allowed.