From 85b31a35155038753aefb442a4581d57993bab73 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Mon, 28 Nov 2016 15:15:50 -0800 Subject: [PATCH 1/4] build: rename build/clean to build/reset --- build/{clean => reset} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/{clean => reset} (100%) diff --git a/build/clean b/build/reset similarity index 100% rename from build/clean rename to build/reset From 8721bc60ff253ec25cd58e65d24c0e0f7265af7d Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Mon, 28 Nov 2016 15:16:16 -0800 Subject: [PATCH 2/4] build: add missing copyright headers --- build/container/external/CMakeLists.txt | 14 ++++++++++++++ build/container/external/packages/boost.cmake | 13 +++++++++++-- build/container/external/packages/cryptopp.cmake | 12 +++++++++++- build/container/external/packages/curl.cmake | 13 +++++++++++-- .../container/external/packages/util-linux.cmake | 12 +++++++++++- build/container/external/packages/zlib.cmake | 12 +++++++++++- build/container/external/test-build.sh | 16 +++++++++++++++- 7 files changed, 84 insertions(+), 8 deletions(-) diff --git a/build/container/external/CMakeLists.txt b/build/container/external/CMakeLists.txt index 9d867c1ce..c45b3aae3 100644 --- a/build/container/external/CMakeLists.txt +++ b/build/container/external/CMakeLists.txt @@ -1,3 +1,17 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + cmake_minimum_required (VERSION 3.0.2) project (external) diff --git a/build/container/external/packages/boost.cmake b/build/container/external/packages/boost.cmake index 62b422ae7..7400a9b83 100644 --- a/build/container/external/packages/boost.cmake +++ b/build/container/external/packages/boost.cmake @@ -1,7 +1,16 @@ - +# Copyright 2016 The Rook Authors. All rights reserved. # -# Config +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # See http://www.boost.org/users/download/#history set(Boost_VERSION 1.62.0) diff --git a/build/container/external/packages/cryptopp.cmake b/build/container/external/packages/cryptopp.cmake index 7ae2526c7..c30cb6c3e 100644 --- a/build/container/external/packages/cryptopp.cmake +++ b/build/container/external/packages/cryptopp.cmake @@ -1,6 +1,16 @@ +# Copyright 2016 The Rook Authors. All rights reserved. # -# Config +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # See https://www.cryptopp.com/downloads.html set(Cryptopp_VERSION 5.6.5) diff --git a/build/container/external/packages/curl.cmake b/build/container/external/packages/curl.cmake index 14768f208..7683a3e35 100644 --- a/build/container/external/packages/curl.cmake +++ b/build/container/external/packages/curl.cmake @@ -1,7 +1,16 @@ - +# Copyright 2016 The Rook Authors. All rights reserved. # -# Config +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # See https://curl.haxx.se/download.html set(Curl_VERSION 7.51.0) diff --git a/build/container/external/packages/util-linux.cmake b/build/container/external/packages/util-linux.cmake index a6c97602a..c5923dffa 100644 --- a/build/container/external/packages/util-linux.cmake +++ b/build/container/external/packages/util-linux.cmake @@ -1,6 +1,16 @@ +# Copyright 2016 The Rook Authors. All rights reserved. # -# Config +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # See http://www.zlib.net set(UtilLinux_VERSION 2.29) diff --git a/build/container/external/packages/zlib.cmake b/build/container/external/packages/zlib.cmake index 405204c03..6a472d71d 100644 --- a/build/container/external/packages/zlib.cmake +++ b/build/container/external/packages/zlib.cmake @@ -1,6 +1,16 @@ +# Copyright 2016 The Rook Authors. All rights reserved. # -# Config +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # See http://www.zlib.net set(Zlib_VERSION 1.2.8) diff --git a/build/container/external/test-build.sh b/build/container/external/test-build.sh index 3b57e55ec..4678ecdb1 100755 --- a/build/container/external/test-build.sh +++ b/build/container/external/test-build.sh @@ -1,4 +1,18 @@ -#/bin/bash -e +#!/bin/bash -e + +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. rm -fr test From e8d1470310ff9423e73622a4a24958911bf0c375 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Tue, 29 Nov 2016 21:50:01 -0800 Subject: [PATCH 3/4] build: build more packages from source we now build all the required packages for rookd from source. also reorganized the external project a bit to support this. Removed STATIC makefile option and replaced with LINKMODE --- .gitignore | 1 + .gitmodules | 1 - Makefile | 53 ++++--- build/Makefile.md | 103 +++++++++++--- build/container/Dockerfile | 129 ++++-------------- build/container/external/CMakeLists.txt | 49 +++++-- .../external/packages/aio/00-makefile.patch | 43 ++++++ .../external/packages/aio/01-noshared.patch | 21 +++ .../external/packages/aio/CMakeLists.txt | 51 +++++++ .../{boost.cmake => boost/CMakeLists.txt} | 9 +- .../CMakeLists.txt} | 43 ++---- .../{curl.cmake => curl/CMakeLists.txt} | 4 +- .../external/packages/expat/CMakeLists.txt | 50 +++++++ .../packages/gperftools/CMakeLists.txt | 48 +++++++ .../external/packages/jemalloc/CMakeLists.txt | 48 +++++++ .../external/packages/openssl/CMakeLists.txt | 57 ++++++++ .../packages/rocksdb/00-portable.patch | 11 ++ .../external/packages/rocksdb/CMakeLists.txt | 49 +++++++ .../external/packages/snappy/CMakeLists.txt | 48 +++++++ .../CMakeLists.txt} | 5 +- .../{zlib.cmake => zlib/CMakeLists.txt} | 4 +- build/container/external/test-build.sh | 22 ++- .../toolchain/aarch64-linux-gnu.cmake | 18 +++ .../external/toolchain/gcc.linux_arm64.cmake | 35 ----- ...{gcc.linux_amd64.cmake => linux-gnu.cmake} | 25 +++- .../external/toolchain/x86_64-linux-gnu.cmake | 18 +++ build/container/version | 2 +- build/makelib/cephd.mk | 9 +- build/makelib/cross.mk | 4 + build/run | 2 + ceph | 2 +- pkg/cephmgr/cephd/cephd.go | 7 +- 32 files changed, 738 insertions(+), 233 deletions(-) create mode 100644 build/container/external/packages/aio/00-makefile.patch create mode 100644 build/container/external/packages/aio/01-noshared.patch create mode 100644 build/container/external/packages/aio/CMakeLists.txt rename build/container/external/packages/{boost.cmake => boost/CMakeLists.txt} (86%) rename build/container/external/packages/{cryptopp.cmake => cryptopp/CMakeLists.txt} (58%) rename build/container/external/packages/{curl.cmake => curl/CMakeLists.txt} (92%) create mode 100644 build/container/external/packages/expat/CMakeLists.txt create mode 100644 build/container/external/packages/gperftools/CMakeLists.txt create mode 100644 build/container/external/packages/jemalloc/CMakeLists.txt create mode 100644 build/container/external/packages/openssl/CMakeLists.txt create mode 100644 build/container/external/packages/rocksdb/00-portable.patch create mode 100644 build/container/external/packages/rocksdb/CMakeLists.txt create mode 100644 build/container/external/packages/snappy/CMakeLists.txt rename build/container/external/packages/{util-linux.cmake => util-linux/CMakeLists.txt} (90%) rename build/container/external/packages/{zlib.cmake => zlib/CMakeLists.txt} (89%) create mode 100644 build/container/external/toolchain/aarch64-linux-gnu.cmake delete mode 100644 build/container/external/toolchain/gcc.linux_arm64.cmake rename build/container/external/toolchain/{gcc.linux_amd64.cmake => linux-gnu.cmake} (58%) create mode 100644 build/container/external/toolchain/x86_64-linux-gnu.cmake diff --git a/.gitignore b/.gitignore index 5df9c10ad..1e829f088 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.glide /.work +/.external /bin /tools diff --git a/.gitmodules b/.gitmodules index e66704e87..c2dbd95ad 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,3 @@ [submodule "ceph"] path = ceph url = https://github.com/rook/ceph.git - branch = wip-embedded-ceph diff --git a/Makefile b/Makefile index 8872fe26b..7fb3a2258 100644 --- a/Makefile +++ b/Makefile @@ -27,13 +27,19 @@ BUILDFLAGS ?= LDFLAGS ?= TAGS ?= -# set to 1 for a completely static build. Otherwise if set to 0 -# a dynamic binary is produced that requires glibc to be installed -STATIC ?= 1 -ifeq ($(STATIC),1) -TAGS += static -else +# if set to 'dynamic' all dependencies are dynamically linked. if +# set to 'static' all dependencies will be statically linked. If set +# to 'stdlib' then the standard library will be dynamically +# linked and everything else will be statically linked. +LINKMODE ?= dynamic +ifeq ($(LINKMODE),dynamic) TAGS += dynamic +else +ifeq ($(LINKMODE),stdlib) +TAGS += stdlib +else +TAGS += static +endif endif # build a position independent executable. This implies dynamic linking @@ -41,8 +47,8 @@ endif # is only supported on Linux. PIE ?= 0 ifeq ($(PIE),1) -ifeq ($(STATIC),1) -$(error PIE only supported with dynamic linking. Set STATIC=0.) +ifeq ($(LINKMODE),static) +$(error PIE only supported with dynamic linking. Set LINKMODE=dynamic or LINKMODE=stdlib.) endif endif @@ -54,7 +60,20 @@ LDFLAGS += -w endif # the memory allocator to use for cephd -ALLOCATOR ?= tcmalloc_minimal +ALLOCATOR ?= tcmalloc +ifeq ($(ALLOCATOR),jemalloc) +CEPHD_ALLOCATOR = jemalloc +TAGS += jemalloc +else +ifeq ($(ALLOCATOR),tcmalloc) +CEPHD_ALLOCATOR = tcmalloc_minimal +TAGS += tcmalloc +else +ifeq ($(ALLOCATOR),libc) +CEPHD_ALLOCATOR = libc +endif +endif +endif # whether to use ccache when building cephd CCACHE ?= 1 @@ -109,7 +128,6 @@ ifeq ($(ROOKD_SUPPORTED),1) CEPHD_DEBUG = $(DEBUG) CEPHD_CCACHE = $(CCACHE) -CEPHD_ALLOCATOR = $(ALLOCATOR) CEPHD_BUILD_DIR = $(WORKDIR)/ceph CEPHD_PLATFORM = $(GOOS)_$(GOARCH) @@ -132,7 +150,7 @@ endif GO_WORK_DIR = $(WORKDIR) GO_BIN_DIR = $(BIN_DIR) -ifeq ($(STATIC),1) +ifeq ($(LINKMODE),static) GO_STATIC_PACKAGES=$(GO_PROJECT) ifeq ($(ROOKD_SUPPORTED),1) GO_STATIC_CGO_PACKAGES=$(GO_PROJECT)/cmd/rookd @@ -186,7 +204,7 @@ clean: go.clean distclean: go.distclean clean build.platform.%: - @$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) build + @$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) CROSSBUILD=1 build build.cross: $(foreach p,$(ALL_PLATFORMS), build.platform.$(p)) @@ -233,11 +251,14 @@ help: @echo ' DEBUG Set to 1 to disable stripping the binaries of.' @echo ' debug information. The default is 0.' @echo ' PIE Set to 1 to build build a position independent' - @echo ' executable. Can not be combined with static.' - @echo ' The default is 0.' + @echo ' executable. Can not be combined with LINKMODE' + @echo ' set to "static". The default is 0.' @echo ' GOOS The OS to build for.' - @echo ' STATIC Set to 1 for static build, 0 for dynamic.' - @echo ' The default is 1.' + @echo ' LINKMODE Set to "dynamic" to link all libraries dynamically.' + @echo ' Set to "stdlib" to link the standard library' + @echo ' dynamically and everything else statically. Set to' + @echo ' "static" to link everything statically. Default is' + @echo ' "dynamic".' @echo ' VERSION The version information compiled into binaries.' @echo ' The default is obtained from git.' @echo ' V Set to 1 enable verbose build. Default is 0.' diff --git a/build/Makefile.md b/build/Makefile.md index 8aef848af..4a200b7ba 100644 --- a/build/Makefile.md +++ b/build/Makefile.md @@ -1,33 +1,104 @@ -Rook is golang binary that uses cgo to interface with embedded Ceph. The Ceph -project is a submodule of Rook and is built from the top level Makefile. +## Link Mode -## Static binary +`rookd` is golang binary that uses cgo to interface with embedded Ceph. The Ceph +project is a submodule of Rook and is built from the top level Makefile. Ceph +has numerous dependencies. We support the following link modes when linking: -This is the easiest option and produces static binaries for `rookd` and `rook`. These -binaries can easily be used inside minimal containers, or run on minimal -linux distributions like CoreOS. +### All Static + +This option produces completely static binaries for `rookd` and `rook`. These +binaries can easily be used inside scratch containers, or on distributions +where dependencies can not be easily satisfied. To build statically we recommend +the you build only through the build container. While its possible to build statically +on the host, you might not be able to satisfy the needed dependencies. ``` -make STATIC=1 +build/run make LINKMODE=static ``` -## Dynamic binary - -If you dont want to distribute a static binary, a dynamically linked binary is -supported. The approach we take is to link most of the glibc binaries dynamically -and the rest of the libraries continue to be linked statically. Ceph has a lot -of dependencies and we take this approach to simplify the distribution. +Here's an example of what the dependencies look like in this mode: ``` -make STATIC=0 +> ldd bin/linux_amd64/rookd + not a dynamic executable +``` + +### All Dynamic (Default) + +This option produces binaries that link dynamically to dependent packages, and +is intended for traditional Linux distributions like Ubuntu or CentOS. All +dependencies would need to be installed on the system for `rookd` to run +correctly. This is the default LINKMODE. + +``` +make LINKMODE=dynamic +``` + +In this mode the system dependencies can vary from one distro to another. Here's an +example of what the dependencies look like on Ubuntu: + +``` +> build/run ldd bin/linux_amd64/rookd + linux-vdso.so.1 => (0x00007ffc053c3000) + libboost_system.so.1.62.0 => /usr/local/lib/x86_64-linux-gnu/libboost_system.so.1.62.0 (0x00007f56f6e33000) + libboost_thread.so.1.62.0 => /usr/local/lib/x86_64-linux-gnu/libboost_thread.so.1.62.0 (0x00007f56f6c0b000) + libboost_iostreams.so.1.62.0 => /usr/local/lib/x86_64-linux-gnu/libboost_iostreams.so.1.62.0 (0x00007f56f69f6000) + libboost_random.so.1.62.0 => /usr/local/lib/x86_64-linux-gnu/libboost_random.so.1.62.0 (0x00007f56f67f0000) + libblkid.so.1 => /usr/local/lib/x86_64-linux-gnu/libblkid.so.1 (0x00007f56f65aa000) + libz.so.1 => /usr/local/lib/x86_64-linux-gnu/libz.so.1 (0x00007f56f6392000) + libsnappy.so.1 => /usr/lib/x86_64-linux-gnu/libsnappy.so.1 (0x00007f56f6184000) + libcrypto++.so.6 => /usr/lib/x86_64-linux-gnu/libcrypto++.so.6 (0x00007f56f5c05000) + libaio.so.1 => /lib/x86_64-linux-gnu/libaio.so.1 (0x00007f56f5a03000) + libcurl.so.4 => /usr/local/lib/x86_64-linux-gnu/libcurl.so.4 (0x00007f56f57cd000) + libexpat.so.1 => /lib/x86_64-linux-gnu/libexpat.so.1 (0x00007f56f5396000) + libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f56f5192000) + libresolv.so.2 => /lib/x86_64-linux-gnu/libresolv.so.2 (0x00007f56f4f77000) + libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f56f4d59000) + libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f56f49d1000) + libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f56f46c8000) + libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f56f44af000) + libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f56f40e8000) + /lib64/ld-linux-x86-64.so.2 (0x0000564ace68d000) + librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f56f3ee0000) + libuuid.so.1 => /usr/local/lib/x86_64-linux-gnu/libuuid.so.1 (0x00007f56f3cdb000) + (more) +``` + +Note the Libraries like libcurl can bring in lots of other dependencies, and vary widely +between distributions. + +### Dynamic Standard Library + +This option produces mostly static binaries, whith just the standard C/C++ libraries +dynamically linked. Its intended for restricted Linux environments like CoreOS when +running directly on the host OS (and not inside a container). + +``` +make LINKMODE=stdlib +``` + +Here's an example of what the dependencies look like in this mode: + +``` +> ldd bin/linux_amd64/rookd + linux-vdso.so.1 => (0x00007fff45fb6000) + libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f5305d4e000) + libresolv.so.2 => /lib/x86_64-linux-gnu/libresolv.so.2 (0x00007f5305b33000) + libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f5305915000) + libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f530558d000) + libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f5305284000) + libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f530506b000) + libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f5304ca4000) + /lib64/ld-linux-x86-64.so.2 (0x000055d3eb36e000) ``` ## Hardened binary (PIE) -You can build a Position Independent executable as follows: +You can build a Position Independent executable. Note that this requires LINKMODE set +to `dynamic` or `stdlib` ``` -make STATIC=0 PIE=1 +make LINKMODE=stdlib PIE=1 ``` ## Switching Allocators diff --git a/build/container/Dockerfile b/build/container/Dockerfile index e6af6382f..d4f5928ba 100644 --- a/build/container/Dockerfile +++ b/build/container/Dockerfile @@ -14,17 +14,14 @@ FROM ubuntu:yakkety MAINTAINER Bassam Tabbara - WORKDIR /build +ENV PATH /usr/local/go/bin:/usr/local/osxcross/bin:$PATH # # install build tools, compilers and cross compilers for all supported platforms # - RUN echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu yakkety main universe restricted" > /etc/apt/sources.list &&\ echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu yakkety-updates main universe restricted" >> /etc/apt/sources.list &&\ - echo "deb [arch=arm64] http://ports.ubuntu.com/ yakkety main universe restricted" >> /etc/apt/sources.list &&\ - echo "deb [arch=arm64] http://ports.ubuntu.com/ yakkety-updates main universe restricted" >> /etc/apt/sources.list &&\ dpkg --add-architecture arm64 && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ @@ -35,7 +32,6 @@ RUN echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu yakkety main univers cmake \ crossbuild-essential-arm64 \ curl \ - cython \ docker.io \ git \ jq \ @@ -43,30 +39,27 @@ RUN echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu yakkety main univers llvm-dev \ mingw-w64 \ patch \ - python \ - libpython-dev:amd64 \ - libpython-dev:arm64 \ rsync \ runit \ sudo \ uuid-dev \ xz-utils \ yasm \ - zip + zip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # install the OSX cross compilers. see https://github.com/tpoechtrager/osxcross RUN OSXCROSS_COMMIT_HASH=86879571d33ff7252a55d777609742561dba9953 && \ - OSXCROSS_SDK=MacOSX10.11.sdk.tar.xz && \ + OSXCROSS_SDK=MacOSX10.11.sdk && \ OSXCROSS_SDK_HASH=d75dddc8d820daf38e90a7c8e9771750a8a4371027a2d658a904cffacbf6b09b && \ git clone https://github.com/quantum/osxcross && \ git -C osxcross checkout -b castle-build ${OSXCROSS_COMMIT_HASH} && \ - curl -fsSL https://castle-cross-build.s3.amazonaws.com/osxcross/${OSXCROSS_SDK} -o osxcross/tarballs/${OSXCROSS_SDK} && \ - echo "${OSXCROSS_SDK_HASH} osxcross/tarballs/${OSXCROSS_SDK}" | sha256sum -c - && \ + curl -fsSL https://castle-cross-build.s3.amazonaws.com/osxcross/${OSXCROSS_SDK}.tar.xz -o osxcross/tarballs/${OSXCROSS_SDK}.tar.xz && \ + echo "${OSXCROSS_SDK_HASH} osxcross/tarballs/${OSXCROSS_SDK}.tar.xz" | sha256sum -c - && \ UNATTENDED=yes OSX_VERSION_MIN=10.6 osxcross/build.sh && \ mv osxcross/target /usr/local/osxcross && \ - rm -rf osxcross - -ENV PATH /usr/local/osxcross/bin:$PATH + rm -rf osxcross && \ + rm -rf /usr/local/osxcross/SDK/${OSXCROSS_SDK}/usr/share # install acbuild and docker2aci RUN ACBUILD_VERSION=0.4.0 && \ @@ -78,24 +71,19 @@ RUN ACBUILD_VERSION=0.4.0 && \ tar -C /usr/local/bin --strip-components=1 -xzf docker2aci-v${DOCKER2ACI_VERSION}.tar.gz && \ rm -f docker2aci-v${DOCKER2ACI_VERSION}.tar.gz -# install go-lang -RUN GO_VERSION=1.7.3 && \ - GO_HASH=508028aac0654e993564b6e2014bf2d4a9751e3b286661b0b0040046cf18028e && \ - curl -fsSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz -o golang.tar.gz && \ - echo "${GO_HASH} golang.tar.gz" | sha256sum -c - && \ - tar -C /usr/local -xzf golang.tar.gz && \ - rm golang.tar.gz - -ENV PATH /usr/local/go/bin:$PATH - -# -# precompile the go standard library for all supported platforms and configurations +# install go-lang and precompile the go standard library for all supported platforms and configurations # there are three configurations that we compile for: # - default: CGO_ENABLED=1, defaults tags # - netgo: CGO_ENABLED=1, enable pure go DNS resolver # - nocgo: CGO_ENABLED=0, defaults tags, pure Go resolver is the default # -RUN \ +RUN GO_VERSION=1.7.4 && \ + GO_HASH=47fda42e46b4c3ec93fa5d4d4cc6a748aa3f9411a2a2b7e08e3a6d80d753ec8b && \ + curl -fsSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz -o golang.tar.gz && \ + echo "${GO_HASH} golang.tar.gz" | sha256sum -c - && \ + tar -C /usr/local -xzf golang.tar.gz && \ + rm golang.tar.gz && \ + \ CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 CC=o64-clang go install -a std && \ CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 CC=o64-clang go install -installsuffix netgo -tags netgo -a std && \ CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 CC=o64-clang go install -installsuffix nocgo -a std && \ @@ -114,82 +102,25 @@ RUN \ CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-gnu-gcc go install -installsuffix race -a std # -# build packages we need for ceph from source on all supported platforms +# build rookd dependencies that are compiled statically directly from source for all supported platforms. # - COPY external /build/external/ RUN set -e;\ - for a in linux_amd64 linux_arm64; do \ - cmake -Hexternal -Bexternal/build/${a} -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_TOOLCHAIN_FILE=`pwd`/external/toolchain/gcc.${a}.cmake ;\ - cmake --build external/build/${a} --target install;\ + for a in x86_64-linux-gnu aarch64-linux-gnu; do \ + cmake \ + -H/build/external \ + -B/build/external/build/${a} \ + -DEXTERNAL_DOWNLOAD_DIR=/build/external/download \ + -DCMAKE_INSTALL_PREFIX=/usr/${a} \ + -DCMAKE_TOOLCHAIN_FILE=/build/external/toolchain/${a}.cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_MESSAGE=never \ + ;\ + cmake --build /build/external/build/${a} --target install -- -j4 ;\ done ;\ - rm -fr external - -RUN set -e;\ - for a in amd64 arm64; do \ - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-overwrite" install -yq --no-install-recommends \ - libatomic-ops-dev:${a} \ - libexpat1-dev:${a} \ - libkeyutils-dev:${a} \ - libsnappy-dev:${a} \ - libssl-dev:${a} \ - libudev-dev:${a} \ - ; done - -# Some of the packages we need for ceph are either not available in yakkety or dont support multi-arch. -# Hack'em to install in a multi-arch way. Hopefully these will get fixed upstream soon. -# Bugs opened here https://bugs.launchpad.net/~bassamtabbara -RUN set -e;\ - curl -fsSLO http://http.us.debian.org/debian/pool/main/libc/libcrypto++/libcrypto++-dev_5.6.4-3_amd64.deb ;\ - curl -fsSLO http://http.us.debian.org/debian/pool/main/libc/libcrypto++/libcrypto++-dev_5.6.4-3_arm64.deb ;\ - curl -fsSLO http://http.us.debian.org/debian/pool/main/libc/libcrypto++/libcrypto++6_5.6.4-3_amd64.deb ;\ - curl -fsSLO http://http.us.debian.org/debian/pool/main/libc/libcrypto++/libcrypto++6_5.6.4-3_arm64.deb ;\ - curl -fsSLO http://archive.ubuntu.com/ubuntu/pool/main/libf/libfcgi/libfcgi-dev_2.4.0-8.4_amd64.deb ;\ - curl -fsSLO http://ports.ubuntu.com/pool/main/libf/libfcgi/libfcgi-dev_2.4.0-8.4_arm64.deb ;\ - curl -fsSLO http://archive.ubuntu.com/ubuntu/pool/main/libf/libfcgi/libfcgi0ldbl_2.4.0-8.4_amd64.deb ;\ - curl -fsSLO http://ports.ubuntu.com/pool/main/libf/libfcgi/libfcgi0ldbl_2.4.0-8.4_arm64.deb ;\ - for a in amd64 arm64; do \ - apt-get download libaio-dev:${a} 2> /dev/null ;\ - apt-get download libaio1:${a} 2> /dev/null ;\ - apt-get download libjemalloc-dev:${a} 2> /dev/null ;\ - apt-get download libjemalloc1:${a} 2> /dev/null ;\ - apt-get download libgoogle-perftools4:${a} 2> /dev/null ;\ - apt-get download libgoogle-perftools-dev:${a} 2> /dev/null ;\ - apt-get download libtcmalloc-minimal4:${a} 2> /dev/null ;\ - apt-get download libunwind-dev:${a} 2> /dev/null ;\ - apt-get download libunwind8-dev:${a} 2> /dev/null ;\ - apt-get download libunwind8:${a} 2> /dev/null ;\ - apt-get download liblzma5:${a} 2> /dev/null; \ - apt-get download liblzma-dev:${a} 2> /dev/null; \ - done - -RUN set -e; \ - for d in *.deb; do \ - dpkg-deb -R ${d} root ;\ - arch=$(cat root/DEBIAN/control | awk '/Architecture/ {printf "%s", $2}') ;\ - case ${arch} in amd64) triple=x86_64-linux-gnu ;; arm64) triple=aarch64-linux-gnu ;; esac ;\ - if test ! -d root/usr/lib/${triple} -a ! -d root/lib/${triple}; then \ - if test -d root/usr/lib; then \ - echo HACK: moving root/usr/lib for package ${d} ;\ - mv root/usr/lib root/usr/lib.rename ;\ - mkdir -p root/usr/lib ;\ - mv root/usr/lib.rename root/usr/lib/${triple}; \ - fi; \ - fi ;\ - if ! grep 'Multi-Arch' root/DEBIAN/control >/dev/null; then \ - echo HACK: setting Multi-Arch: same for package ${d}; \ - sed -i '/^Architecture:.*$/a Multi-Arch: same' root/DEBIAN/control; \ - fi ;\ - dpkg-deb -b root ${d} ;\ - rm -fr root; \ - done ;\ - dpkg -i --force-overwrite *.deb ;\ - rm -f *.deb - -# cleanup apt-get -RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + rm -fr /build/external COPY entrypoint.sh rsyncd.sh /build/ - ENTRYPOINT [ "/build/entrypoint.sh" ] diff --git a/build/container/external/CMakeLists.txt b/build/container/external/CMakeLists.txt index c45b3aae3..25a1c319d 100644 --- a/build/container/external/CMakeLists.txt +++ b/build/container/external/CMakeLists.txt @@ -22,7 +22,7 @@ include(ProcessorCount) set(EXTERNAL_ROOT ${CMAKE_BINARY_DIR}) # set external download/cache dir -set(EXTERNAL_DOWNLOAD_DIR ${PROJECT_SOURCE_DIR}/download) +option(EXTERNAL_DOWNLOAD_DIR "where to cache the download packages" ${CMAKE_BINARY_DIR}/download) # set the cross triple used set(EXTERNAL_CROSS_TRIPLE ${CROSS_TRIPLE}) @@ -35,23 +35,56 @@ set(EXTERNAL_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}) option(EXTERNAL_LOGGING "whether to log all external build output to files" ON) # set the processor count +# FIX: this should really be done at runtime with make -j4. Boost requires this +# now since it goes inside ExternalProject_Add ProcessorCount(EXTERNAL_PARALLEL_LEVEL) +# set build type and compile lags +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE DEBUG) +endif() + +string(TOUPPER "${CMAKE_BUILD_TYPE}" build_type_upper) +set(EXTERNAL_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${build_type_upper}}") +set(EXTERNAL_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${build_type_upper}}") + message(STATUS "External: Building for ${CROSS_TRIPLE}") +message(STATUS "External: Build Type: ${CMAKE_BUILD_TYPE}") +message(STATUS "External: CFLAGS: ${EXTERNAL_C_FLAGS}") +message(STATUS "External: CXXFLAGS: ${EXTERNAL_CXX_FLAGS}") message(STATUS "External: Installing to ${EXTERNAL_INSTALL_DIR}") # -# Build the external projects in order of dependencies +# Build the external projects # -include(packages/zlib.cmake) -include(packages/boost.cmake) -include(packages/curl.cmake) -include(packages/util-linux.cmake) +add_subdirectory(packages/zlib) +add_subdirectory(packages/aio) +add_subdirectory(packages/boost) +add_subdirectory(packages/cryptopp) +add_subdirectory(packages/curl) +add_subdirectory(packages/expat) +add_subdirectory(packages/gperftools) +add_subdirectory(packages/jemalloc) +add_subdirectory(packages/openssl) +add_subdirectory(packages/snappy) +add_subdirectory(packages/util-linux) +add_subdirectory(packages/rocksdb) # # Copy include files and libraries to the destination # -install(DIRECTORY ${EXTERNAL_ROOT}/include DESTINATION ${EXTERNAL_INSTALL_DIR} USE_SOURCE_PERMISSIONS) -install(DIRECTORY ${EXTERNAL_ROOT}/lib DESTINATION ${EXTERNAL_INSTALL_DIR} USE_SOURCE_PERMISSIONS) +install( + DIRECTORY ${EXTERNAL_ROOT}/include + DESTINATION ${EXTERNAL_INSTALL_DIR} + USE_SOURCE_PERMISSIONS) + +install( + DIRECTORY ${EXTERNAL_ROOT}/lib + DESTINATION ${EXTERNAL_INSTALL_DIR} + USE_SOURCE_PERMISSIONS + PATTERN "pkgconfig" EXCLUDE + PATTERN "*.la" EXCLUDE) + +# TODO: strip symbols into separate dbg \ No newline at end of file diff --git a/build/container/external/packages/aio/00-makefile.patch b/build/container/external/packages/aio/00-makefile.patch new file mode 100644 index 000000000..d3b6b0c09 --- /dev/null +++ b/build/container/external/packages/aio/00-makefile.patch @@ -0,0 +1,43 @@ +--- src/Makefile 2016-11-30 09:59:12.889526296 -0800 ++++ src/Makefile 2016-11-30 10:00:46.812341855 -0800 +@@ -2,12 +2,12 @@ + includedir=$(prefix)/include + libdir=$(prefix)/lib + +-CFLAGS ?= -g -fomit-frame-pointer -O2 +-CFLAGS += -nostdlib -nostartfiles -Wall -I. -fPIC +-SO_CFLAGS=-shared $(CFLAGS) +-L_CFLAGS=$(CFLAGS) +-LINK_FLAGS= +-LINK_FLAGS+=$(LDFLAGS) ++MK_CPPFLAGS=-I. ++MK_CFLAGS=-nostdlib -nostartfiles -fomit-frame-pointer -fPIC $(CFLAGS) ++CFLAGS=-Wall -g -O2 ++SO_CFLAGS=-shared $(MK_CFLAGS) ++L_CFLAGS=$(MK_CFLAGS) ++MK_LDFLAGS=-Wl,--as-needed -lgcc -lc $(LDFLAGS) + + soname=libaio.so.1 + minor=0 +@@ -37,10 +37,10 @@ + $(libaio_objs) $(libaio_sobjs): libaio.h vsys_def.h + + %.os: %.c +- $(CC) $(SO_CFLAGS) -c -o $@ $< ++ $(CC) $(MK_CPPFLAGS) $(SO_CFLAGS) -c -o $@ $< + + %.ol: %.c +- $(CC) $(L_CFLAGS) -c -o $@ $< ++ $(CC) $(MK_CPPFLAGS) $(L_CFLAGS) -c -o $@ $< + + AR ?= ar + RANLIB ?= ranlib +@@ -50,7 +50,7 @@ + $(RANLIB) libaio.a + + $(libname): $(libaio_sobjs) libaio.map +- $(CC) $(SO_CFLAGS) -Wl,--version-script=libaio.map -Wl,-soname=$(soname) -o $@ $(libaio_sobjs) $(LINK_FLAGS) ++ $(CC) $(SO_CFLAGS) -Wl,--version-script=libaio.map -Wl,-soname=$(soname) -o $@ $(libaio_sobjs) $(MK_CPPFLAGS) + + install: $(all_targets) + install -D -m 644 libaio.h $(includedir)/libaio.h diff --git a/build/container/external/packages/aio/01-noshared.patch b/build/container/external/packages/aio/01-noshared.patch new file mode 100644 index 000000000..f48e9f732 --- /dev/null +++ b/build/container/external/packages/aio/01-noshared.patch @@ -0,0 +1,21 @@ +--- src/Makefile 2016-11-30 10:44:03.932450351 -0800 ++++ src/Makefile 2016-11-30 10:44:24.620310628 -0800 +@@ -13,7 +13,7 @@ + minor=0 + micro=1 + libname=$(soname).$(minor).$(micro) +-all_targets += libaio.a $(libname) ++all_targets += libaio.a + + all: $(all_targets) + +@@ -55,9 +55,6 @@ + install: $(all_targets) + install -D -m 644 libaio.h $(includedir)/libaio.h + install -D -m 644 libaio.a $(libdir)/libaio.a +- install -D -m 755 $(libname) $(libdir)/$(libname) +- ln -sf $(libname) $(libdir)/$(soname) +- ln -sf $(libname) $(libdir)/libaio.so + + $(libaio_objs): libaio.h + diff --git a/build/container/external/packages/aio/CMakeLists.txt b/build/container/external/packages/aio/CMakeLists.txt new file mode 100644 index 000000000..9499038aa --- /dev/null +++ b/build/container/external/packages/aio/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://git.fedorahosted.org/cgit/libaio.git/ +set(Aio_VERSION 0.3.110-1) +set(Aio_URL https://git.fedorahosted.org/cgit/libaio.git/snapshot/libaio-libaio-${Aio_VERSION}.tar.gz) + +message(STATUS "External: Building libaio ${Aio_VERSION}") + +# +# Build +# + +set(Aio_CFLAGS ${EXTERNAL_C_FLAGS}) +set(Aio_TOOLCHAIN CC=${CMAKE_C_COMPILER} AR=${CMAKE_AR} RANLIB=${CMAKE_RANLIB}) + +set(Aio_Config_Args + --disable-shared + ) + +ExternalProject_Add(aio + PREFIX ${EXTERNAL_ROOT} + + URL ${Aio_URL} + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND + patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/00-makefile.patch + COMMAND patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/01-noshared.patch + CONFIGURE_COMMAND true + BUILD_COMMAND $(MAKE) CFLAGS=${Aio_CFLAGS} ${Aio_TOOLCHAIN} + INSTALL_COMMAND $(MAKE) prefix= install + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) diff --git a/build/container/external/packages/boost.cmake b/build/container/external/packages/boost/CMakeLists.txt similarity index 86% rename from build/container/external/packages/boost.cmake rename to build/container/external/packages/boost/CMakeLists.txt index 7400a9b83..3400c346a 100644 --- a/build/container/external/packages/boost.cmake +++ b/build/container/external/packages/boost/CMakeLists.txt @@ -17,7 +17,7 @@ set(Boost_VERSION 1.62.0) set(Boost_VERSION_FILE 1_62_0) set(Boost_URL https://sourceforge.net/projects/boost/files/boost/${Boost_VERSION}/boost_${Boost_VERSION_FILE}.tar.bz2) set(Boost_URL_SHA256 36c96b0f6155c98404091d8ceb48319a28279ca0333fba1ad8611eb90afb2ca0) -set(Boost_COMPONENTS thread system regex random program_options date_time iostreams) +set(Boost_COMPONENTS thread system random iostreams) message(STATUS "External: Building Boost ${Boost_VERSION} COMPONENTS ${Boost_COMPONENTS}") @@ -37,10 +37,11 @@ else() message(FATAL_ERROR "Unknown compiler ${CMAKE_CXX_COMPILER_ID}.") endif() -set(Boost_CFLAGS "-fPIC -O2") +set(Boost_CFLAGS ${EXTERNAL_C_FLAGS}) set(Boost_VERBOSE_ARGS -d+2 --debug-configuration) ExternalProject_Add(boost + DEPENDS zlib PREFIX ${EXTERNAL_ROOT} URL ${Boost_URL} @@ -50,7 +51,7 @@ ExternalProject_Add(boost BUILD_IN_SOURCE 1 PATCH_COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_BINARY_DIR}/user-config.jam" "/tools/build/src/user-config.jam" - CONFIGURE_COMMAND ./bootstrap.sh --prefix= --libdir=/lib/${EXTERNAL_CROSS_TRIPLE} --with-libraries=${Boost_BUILD_COMPONENTS} + CONFIGURE_COMMAND ./bootstrap.sh --prefix= --with-libraries=${Boost_BUILD_COMPONENTS} BUILD_COMMAND true INSTALL_COMMAND ./b2 ${Boost_VERBOSE_ARGS} @@ -60,7 +61,7 @@ ExternalProject_Add(boost link=static threading=multi "include=/include" - "linkflags=-L/lib/${EXTERNAL_CROSS_TRIPLE}" + "linkflags=-L/lib" cxxflags=${Boost_CFLAGS} install diff --git a/build/container/external/packages/cryptopp.cmake b/build/container/external/packages/cryptopp/CMakeLists.txt similarity index 58% rename from build/container/external/packages/cryptopp.cmake rename to build/container/external/packages/cryptopp/CMakeLists.txt index c30cb6c3e..f3696441f 100644 --- a/build/container/external/packages/cryptopp.cmake +++ b/build/container/external/packages/cryptopp/CMakeLists.txt @@ -23,49 +23,24 @@ message(STATUS "External: Building Cryptopp ${Cryptopp_VERSION}") # Build # -set(Curl_Config_Args - --disable-cookies - --disable-crypto-auth - --disable-dict - --disable-ftp - --disable-gopher - --disable-imap - --disable-ldap - --disable-manual - --disable-pop3 - --disable-rtsp - --disable-shared - --disable-smb - --disable-smtp - --disable-telnet - --disable-tftp - --disable-unix-sockets - --with-ssl - --without-gssapi - --without-libssh2 - --without-nss - --without-winidn - ) +set(Cryptopp_CXXFLAGS ${EXTERNAL_CXX_FLAGS}) +set(Cryptopp_TOOLCHAIN CXX=${CMAKE_CXX_COMPILER} AR=${CMAKE_AR} RANLIB=${CMAKE_RANLIB}) -set(Curl_MAKE_ARGS -j${EXTERNAL_PARALLEL_LEVEL}) +set(Cryptopp_MAKE_ARGS) -if(EXTERNAL_VERBOSE) - list(APPEND Curl_MAKE_ARGS V=1) -endif() - -ExternalProject_Add(curl +ExternalProject_Add(cryptopp PREFIX ${EXTERNAL_ROOT} - URL ${Curl_URL} - URL_HASH SHA256=${Curl_URL_SHA256} + URL ${Cryptopp_URL} + URL_HASH SHA256=${Cryptopp_URL_SHA256} DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} BUILD_IN_SOURCE 1 PATCH_COMMAND true - CONFIGURE_COMMAND ./configure --prefix= ${Curl_Config_Args} --host=${EXTERNAL_CROSS_TRIPLE} --libdir=/lib/${EXTERNAL_CROSS_TRIPLE} - BUILD_COMMAND make ${Curl_MAKE_ARGS} - INSTALL_COMMAND make ${Curl_MAKE_ARGS} install + CONFIGURE_COMMAND true + BUILD_COMMAND $(MAKE) CXXFLAGS=${Cryptopp_CXXFLAGS} ${Cryptopp_TOOLCHAIN} static + INSTALL_COMMAND $(MAKE) PREFIX= install LOG_DOWNLOAD ${EXTERNAL_LOGGING} LOG_PATCH ${EXTERNAL_LOGGING} diff --git a/build/container/external/packages/curl.cmake b/build/container/external/packages/curl/CMakeLists.txt similarity index 92% rename from build/container/external/packages/curl.cmake rename to build/container/external/packages/curl/CMakeLists.txt index 7683a3e35..b23b6f600 100644 --- a/build/container/external/packages/curl.cmake +++ b/build/container/external/packages/curl/CMakeLists.txt @@ -23,7 +23,7 @@ message(STATUS "External: Building Curl ${Curl_VERSION}") # Build # -set(Curl_CFLAGS "-fPIC -O2") +set(Curl_CFLAGS ${EXTERNAL_C_FLAGS}) set(Curl_Config_Args --disable-cookies @@ -59,7 +59,7 @@ ExternalProject_Add(curl BUILD_IN_SOURCE 1 PATCH_COMMAND true - CONFIGURE_COMMAND ./configure CFLAGS=${Curl_CFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} --libdir=/lib/${EXTERNAL_CROSS_TRIPLE} ${Curl_Config_Args} + CONFIGURE_COMMAND ./configure CFLAGS=${Curl_CFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} ${Curl_Config_Args} BUILD_COMMAND $(MAKE) INSTALL_COMMAND $(MAKE) install diff --git a/build/container/external/packages/expat/CMakeLists.txt b/build/container/external/packages/expat/CMakeLists.txt new file mode 100644 index 000000000..67c54a41a --- /dev/null +++ b/build/container/external/packages/expat/CMakeLists.txt @@ -0,0 +1,50 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://expat.sourceforge.net +set(Expat_VERSION 2.2.0) +set(Expat_URL https://sourceforge.net/projects/expat/files/expat/${Expat_VERSION}/expat-${Expat_VERSION}.tar.bz2) +set(Expat_URL_SHA1 8453bc52324be4c796fd38742ec48470eef358b3) + +message(STATUS "External: Building util-linux ${Expat_VERSION}") + +# +# Build +# + +set(Expat_CFLAGS ${EXTERNAL_C_FLAGS}) + +set(Expat_Config_Args + --disable-shared + ) + +ExternalProject_Add(expat + PREFIX ${EXTERNAL_ROOT} + + URL ${Expat_URL} + URL_HASH SHA1=${Expat_URL_SHA1} + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND true + CONFIGURE_COMMAND ./configure CFLAGS=${Expat_CFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} ${Expat_Config_Args} + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) diff --git a/build/container/external/packages/gperftools/CMakeLists.txt b/build/container/external/packages/gperftools/CMakeLists.txt new file mode 100644 index 000000000..e5a0d7a04 --- /dev/null +++ b/build/container/external/packages/gperftools/CMakeLists.txt @@ -0,0 +1,48 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://github.com/gperftools/gperftools +set(Gperftools_VERSION 2.5) +set(Gperftools_URL https://github.com/gperftools/gperftools/releases/download/gperftools-${Gperftools_VERSION}/gperftools-${Gperftools_VERSION}.tar.gz) + +message(STATUS "External: Building gperftools ${Gperftools_VERSION}") + +# +# Build +# + +set(Gperftools_CXXFLAGS ${EXTERNAL_CXX_FLAGS}) + +set(Gperftools_Config_Args + --disable-shared + ) + +ExternalProject_Add(gperftools + PREFIX ${EXTERNAL_ROOT} + + URL ${Gperftools_URL} + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND true + CONFIGURE_COMMAND ./configure CXXFLAGS=${Gperftools_CXXFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} ${Gperftools_Config_Args} + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) diff --git a/build/container/external/packages/jemalloc/CMakeLists.txt b/build/container/external/packages/jemalloc/CMakeLists.txt new file mode 100644 index 000000000..6cf0a5395 --- /dev/null +++ b/build/container/external/packages/jemalloc/CMakeLists.txt @@ -0,0 +1,48 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://github.com/google/snappy +set(Jemalloc_VERSION 4.3.1) +set(Jemalloc_URL https://github.com/jemalloc/jemalloc/releases/download/${Jemalloc_VERSION}/jemalloc-${Jemalloc_VERSION}.tar.bz2) + +message(STATUS "External: Building jemalloc ${Jemalloc_VERSION}") + +# +# Build +# + +#FIX: strip jemalloc since we are unable to remove the g3. + +set(Jemalloc_CFLAGS ${EXTERNAL_C_FLAGS}) + +set(Jemalloc_Config_Args) + +ExternalProject_Add(jemalloc + PREFIX ${EXTERNAL_ROOT} + + URL ${Jemalloc_URL} + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND true + CONFIGURE_COMMAND ./configure EXTRA_CFLAGS=${Jemalloc_CFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} ${Jemalloc_Config_Args} + BUILD_COMMAND $(MAKE) build_lib_static + INSTALL_COMMAND $(MAKE) install_include install_lib_static + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) diff --git a/build/container/external/packages/openssl/CMakeLists.txt b/build/container/external/packages/openssl/CMakeLists.txt new file mode 100644 index 000000000..b6a8e5b45 --- /dev/null +++ b/build/container/external/packages/openssl/CMakeLists.txt @@ -0,0 +1,57 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://www.openssl.org/source/ +set(Openssl_VERSION 1.1.0c) +set(Openssl_URL https://www.openssl.org/source/openssl-${Openssl_VERSION}.tar.gz) +set(Openssl_URL_SHA256 fc436441a2e05752d31b4e46115eb89709a28aef96d4fe786abe92409b2fd6f5) + +message(STATUS "External: Building Cryptopp ${Openssl_VERSION}") + +# +# Build +# + +set(Openssl_CFLAGS ${EXTERNAL_CXX_FLAGS}) + +if (${EXTERNAL_CROSS_TRIPLE} STREQUAL "x86_64-linux-gnu") + set(Openssl_Target linux-x86_64) +elseif(${EXTERNAL_CROSS_TRIPLE} STREQUAL "aarch64-linux-gnu") + set(Openssl_Target linux-aarch64) +else() + message(FATAL_ERROR "openssl: unsupported platform ${EXTERNAL_CROSS_TRIPLE}") +endif() + +ExternalProject_Add(openssl + PREFIX ${EXTERNAL_ROOT} + + URL ${Openssl_URL} + URL_HASH SHA256=${Openssl_URL_SHA256} + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND true + CONFIGURE_COMMAND /Configure --prefix= --cross-compile-prefix=${EXTERNAL_CROSS_TRIPLE}- no-shared ${Openssl_Target} + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) + + + diff --git a/build/container/external/packages/rocksdb/00-portable.patch b/build/container/external/packages/rocksdb/00-portable.patch new file mode 100644 index 000000000..501e4b60d --- /dev/null +++ b/build/container/external/packages/rocksdb/00-portable.patch @@ -0,0 +1,11 @@ +--- Makefile 2016-12-09 17:53:47.231716988 -0800 ++++ Makefile 2016-12-09 17:54:20.954446804 -0800 +@@ -137,7 +137,7 @@ + + AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) + # detect what platform we're building on +-dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) ++dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; export PORTABLE="$(PORTABLE)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk")) + # this file is generated by the previous line to set build flags and sources + include make_config.mk + CLEAN_FILES += make_config.mk diff --git a/build/container/external/packages/rocksdb/CMakeLists.txt b/build/container/external/packages/rocksdb/CMakeLists.txt new file mode 100644 index 000000000..be3523054 --- /dev/null +++ b/build/container/external/packages/rocksdb/CMakeLists.txt @@ -0,0 +1,49 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://github.com/google/snappy +# set(Rocksdb_VERSION 4.11.2) +# set(Rocksdb_URL https://github.com/facebook/rocksdb/archive/v${Rocksdb_VERSION}.tar.gz) + +message(STATUS "External: Building rocksdb ${Rocksdb_VERSION}") + +# +# Build +# + +set(Rocksdb_CXXFLAGS ${EXTERNAL_CXX_FLAGS}) + +set(Rocksdb_Config_Args) + +ExternalProject_Add(rocksdb + DEPENDS zlib snappy + PREFIX ${EXTERNAL_ROOT} + + GIT_REPOSITORY https://github.com/ceph/rocksdb.git + GIT_TAG a0deec960f3a8190831c673e5ba998fe6fb7ea90 + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND + patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/00-portable.patch + CONFIGURE_COMMAND true + BUILD_COMMAND $(MAKE) PORTABLE=1 DISABLE_JEMALLOC=1 "CXX=${CMAKE_CXX_COMPILER}" "EXTRA_CXXFLAGS=${Rocksdb_CXXFLAGS} -I/include" static_lib + INSTALL_COMMAND $(MAKE) INSTALL_PATH= install-static + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) diff --git a/build/container/external/packages/snappy/CMakeLists.txt b/build/container/external/packages/snappy/CMakeLists.txt new file mode 100644 index 000000000..d949daed8 --- /dev/null +++ b/build/container/external/packages/snappy/CMakeLists.txt @@ -0,0 +1,48 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See https://github.com/google/snappy +set(Snappy_VERSION 1.1.3) +set(Snappy_URL https://github.com/google/snappy/releases/download/${Snappy_VERSION}/snappy-${Snappy_VERSION}.tar.gz) + +message(STATUS "External: Building snappy ${Snappy_VERSION}") + +# +# Build +# + +set(Snappy_CXXFLAGS ${EXTERNAL_CXX_FLAGS}) + +set(Snappy_Config_Args + --disable-shared + ) + +ExternalProject_Add(snappy + PREFIX ${EXTERNAL_ROOT} + + URL ${Snappy_URL} + + DOWNLOAD_DIR ${EXTERNAL_DOWNLOAD_DIR} + BUILD_IN_SOURCE 1 + + PATCH_COMMAND true + CONFIGURE_COMMAND ./configure CXXFLAGS=${Snappy_CXXFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} ${Snappy_Config_Args} + BUILD_COMMAND $(MAKE) + INSTALL_COMMAND $(MAKE) install + + LOG_DOWNLOAD ${EXTERNAL_LOGGING} + LOG_PATCH ${EXTERNAL_LOGGING} + LOG_CONFIGURE ${EXTERNAL_LOGGING} + LOG_BUILD ${EXTERNAL_LOGGING} + LOG_INSTALL ${EXTERNAL_LOGGING}) diff --git a/build/container/external/packages/util-linux.cmake b/build/container/external/packages/util-linux/CMakeLists.txt similarity index 90% rename from build/container/external/packages/util-linux.cmake rename to build/container/external/packages/util-linux/CMakeLists.txt index c5923dffa..a9e69a2de 100644 --- a/build/container/external/packages/util-linux.cmake +++ b/build/container/external/packages/util-linux/CMakeLists.txt @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# See http://www.zlib.net set(UtilLinux_VERSION 2.29) set(UtilLinux_URL https://www.kernel.org/pub/linux/utils/util-linux/v${UtilLinux_VERSION}/util-linux-${UtilLinux_VERSION}.tar.gz) set(UtilLinux_URL_SHA256 36d2fe6fcc962b6b4354ecd427ad766bc4ad29581f83afe67c3eadbdc5245a1f) @@ -23,7 +22,7 @@ message(STATUS "External: Building util-linux ${UtilLinux_VERSION}") # Build # -set(UtilLinux_CFLAGS "-fPIC -O2") +set(UtilLinux_CFLAGS ${EXTERNAL_C_FLAGS}) set(UtilLinux_Config_Args --disable-all-programs @@ -51,7 +50,7 @@ ExternalProject_Add(util-linux BUILD_IN_SOURCE 1 PATCH_COMMAND true - CONFIGURE_COMMAND ./configure CFLAGS=${UtilLinux_CFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} --libdir=/lib/${EXTERNAL_CROSS_TRIPLE} ${UtilLinux_Config_Args} + CONFIGURE_COMMAND ./configure CFLAGS=${UtilLinux_CFLAGS} --prefix= --host=${EXTERNAL_CROSS_TRIPLE} ${UtilLinux_Config_Args} BUILD_COMMAND $(MAKE) libblkid.la libuuid.la INSTALL_COMMAND $(MAKE) install diff --git a/build/container/external/packages/zlib.cmake b/build/container/external/packages/zlib/CMakeLists.txt similarity index 89% rename from build/container/external/packages/zlib.cmake rename to build/container/external/packages/zlib/CMakeLists.txt index 6a472d71d..c6fd4efed 100644 --- a/build/container/external/packages/zlib.cmake +++ b/build/container/external/packages/zlib/CMakeLists.txt @@ -23,7 +23,7 @@ message(STATUS "External: Building Zlib ${Zlib_VERSION}") # Build # -set(Zlib_CFLAGS "-fPIC -O2") +set(Zlib_CFLAGS ${EXTERNAL_C_FLAGS}) set(Zlib_Config_Args --static @@ -39,7 +39,7 @@ ExternalProject_Add(zlib BUILD_IN_SOURCE 1 PATCH_COMMAND true - CONFIGURE_COMMAND bash -c "CHOST=${EXTERNAL_CROSS_TRIPLE} CFLAGS='${Zlib_CFLAGS}' ./configure --prefix= --libdir=/lib/${EXTERNAL_CROSS_TRIPLE} ${Zlib_Config_Args}" + CONFIGURE_COMMAND bash -c "CHOST=${EXTERNAL_CROSS_TRIPLE} CFLAGS='${Zlib_CFLAGS}' ./configure --prefix= ${Zlib_Config_Args}" BUILD_COMMAND $(MAKE) INSTALL_COMMAND $(MAKE) install diff --git a/build/container/external/test-build.sh b/build/container/external/test-build.sh index 4678ecdb1..e11bc0e60 100755 --- a/build/container/external/test-build.sh +++ b/build/container/external/test-build.sh @@ -14,10 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -rm -fr test +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +workdir=${scriptdir}/../../../.external +builddir=${workdir}/build +installdir=${workdir}/install +downloaddir=${workdir}/download -for p in linux_amd64 linux_arm64; do - rm -fr build - cmake -H. -Bbuild -DCMAKE_INSTALL_PREFIX=`pwd`/test -DCMAKE_TOOLCHAIN_FILE=`pwd`/toolchain/gcc.${p}.cmake -DEXTERNAL_LOGGING=OFF - make -C build -j4 VERBOSE=1 V=1 install +for p in x86_64-linux-gnu aarch64-linux-gnu; do + rm -fr ${builddir}/${p} + cmake \ + -H${scriptdir} \ + -B${builddir}/${p} \ + -DEXTERNAL_DOWNLOAD_DIR=${downloaddir} \ + -DCMAKE_INSTALL_PREFIX=${installdir}/${p} \ + -DCMAKE_TOOLCHAIN_FILE=${scriptdir}/toolchain/${p}.cmake \ + -DEXTERNAL_LOGGING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + cmake --build ${builddir}/${p} --target install -- V=1 VERBOSE=1 -j3 ;\ done diff --git a/build/container/external/toolchain/aarch64-linux-gnu.cmake b/build/container/external/toolchain/aarch64-linux-gnu.cmake new file mode 100644 index 000000000..26484a59b --- /dev/null +++ b/build/container/external/toolchain/aarch64-linux-gnu.cmake @@ -0,0 +1,18 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(CROSS_TRIPLE "aarch64-linux-gnu") + +include(${CMAKE_CURRENT_LIST_DIR}/linux-gnu.cmake) diff --git a/build/container/external/toolchain/gcc.linux_arm64.cmake b/build/container/external/toolchain/gcc.linux_arm64.cmake deleted file mode 100644 index 0d80fcfb6..000000000 --- a/build/container/external/toolchain/gcc.linux_arm64.cmake +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2016 The Rook Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR aarch64) - -set(CROSS_TRIPLE "aarch64-linux-gnu") - -set(CMAKE_C_COMPILER "/usr/bin/${CROSS_TRIPLE}-gcc" CACHE PATH "C compiler") -set(CMAKE_CXX_COMPILER "/usr/bin/${CROSS_TRIPLE}-g++" CACHE PATH "C++ compiler") -set(CMAKE_ASM_COMPILER "/usr/bin/${CROSS_TRIPLE}-gcc" CACHE PATH "assembler") -set(CMAKE_STRIP "/usr/bin/${CROSS_TRIPLE}-strip" CACHE PATH "strip") -set(CMAKE_AR "/usr/bin/${CROSS_TRIPLE}-ar" CACHE PATH "archive") -set(CMAKE_LINKER "/usr/bin/${CROSS_TRIPLE}-ld" CACHE PATH "linker") -set(CMAKE_NM "/usr/bin/${CROSS_TRIPLE}-nm" CACHE PATH "nm") -set(CMAKE_OBJCOPY "/usr/bin/${CROSS_TRIPLE}-objcopy" CACHE PATH "objcopy") -set(CMAKE_OBJDUMP "/usr/bin/${CROSS_TRIPLE}-objdump" CACHE PATH "objdump") -set(CMAKE_RANLIB "/usr/bin/${CROSS_TRIPLE}-ranlib" CACHE PATH "ranlib") - -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/build/container/external/toolchain/gcc.linux_amd64.cmake b/build/container/external/toolchain/linux-gnu.cmake similarity index 58% rename from build/container/external/toolchain/gcc.linux_amd64.cmake rename to build/container/external/toolchain/linux-gnu.cmake index 9c4e8a111..33ee4551c 100644 --- a/build/container/external/toolchain/gcc.linux_amd64.cmake +++ b/build/container/external/toolchain/linux-gnu.cmake @@ -14,9 +14,6 @@ set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR x86_64) - -set(CROSS_TRIPLE "x86_64-linux-gnu") set(CMAKE_C_COMPILER "/usr/bin/${CROSS_TRIPLE}-gcc" CACHE PATH "C compiler") set(CMAKE_CXX_COMPILER "/usr/bin/${CROSS_TRIPLE}-g++" CACHE PATH "C++ compiler") @@ -29,7 +26,29 @@ set(CMAKE_OBJCOPY "/usr/bin/${CROSS_TRIPLE}-objcopy" CACHE PATH "objcopy") set(CMAKE_OBJDUMP "/usr/bin/${CROSS_TRIPLE}-objdump" CACHE PATH "objdump") set(CMAKE_RANLIB "/usr/bin/${CROSS_TRIPLE}-ranlib" CACHE PATH "ranlib") +set(CMAKE_FIND_ROOT_PATH /usr/${CROSS_TRIPLE}) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +# the toolchain file is included multiple times during config. +# this ensures that the flags are set only once +if(DEFINED TOOLCHAIN_FLAGS_CONFIGURED) + return() +else() + set(TOOLCHAIN_FLAGS_CONFIGURED 1) +endif() + +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g" CACHE STRING "c Debug flags" FORCE) +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g" CACHE STRING "c++ Debug flags" FORCE) +set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -DNDEBUG" CACHE STRING "c Release flags" FORCE) +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -DNDEBUG" CACHE STRING "c++ Release flags" FORCE) + +if (CMAKE_POSITION_INDEPENDENT_CODE) + set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fPIC" CACHE STRING "c Debug flags" FORCE) + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fPIC" CACHE STRING "c++ Debug flags" FORCE) + set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -fPIC" CACHE STRING "c Release flags" FORCE) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fPIC" CACHE STRING "c++ Release flags" FORCE) +endif() diff --git a/build/container/external/toolchain/x86_64-linux-gnu.cmake b/build/container/external/toolchain/x86_64-linux-gnu.cmake new file mode 100644 index 000000000..2cefb4800 --- /dev/null +++ b/build/container/external/toolchain/x86_64-linux-gnu.cmake @@ -0,0 +1,18 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(CROSS_TRIPLE "x86_64-linux-gnu") + +include(${CMAKE_CURRENT_LIST_DIR}/linux-gnu.cmake) diff --git a/build/container/version b/build/container/version index 341cf11fa..373f8c6f0 100644 --- a/build/container/version +++ b/build/container/version @@ -1 +1 @@ -0.2.0 \ No newline at end of file +0.2.3 \ No newline at end of file diff --git a/build/makelib/cephd.mk b/build/makelib/cephd.mk index 6dc3d6c8a..1e32c42b0 100644 --- a/build/makelib/cephd.mk +++ b/build/makelib/cephd.mk @@ -41,6 +41,7 @@ CEPHD_CMAKE += \ -DWITH_FUSE=OFF \ -DWITH_NSS=OFF \ -DWITH_SYSTEM_BOOST=ON \ + -DWITH_SYSTEM_ROCKSDB=ON \ -DUSE_CRYPTOPP=ON \ -DWITH_LEVELDB=OFF \ -DWITH_XFS=OFF \ @@ -49,12 +50,18 @@ CEPHD_CMAKE += \ -DWITH_PROFILER=OFF \ -DWITH_LTTNG=OFF \ -DWITH_MGR=OFF \ + -DWITH_RADOSGW_FCGI_FRONTEND=OFF \ + -DWITH_RADOSGW_ASIO_FRONTEND=OFF \ -DWITH_PYTHON3=OFF ifeq ($(CEPHD_CCACHE),1) CEPHD_CMAKE += -DWITH_CCACHE=ON endif +ifeq ($(CROSSBUILD),1) +CEPHD_CMAKE += -DCMAKE_TOOLCHAIN_FILE=$(abspath build/container/external/toolchain/$(CROSS_TRIPLE).cmake) +endif + # ==================================================================================== # Targets @@ -63,7 +70,7 @@ cephd.config: @mkdir -p $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM) @echo "$(CEPHD_CMAKE)" > $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM)/cephd.cmake.new @if test ! -f $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM)/cephd.cmake || ! diff $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM)/cephd.cmake.new $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM)/cephd.cmake > /dev/null; then \ - cd $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM) && cmake $(CEPHD_CMAKE) -DCMAKE_TOOLCHAIN_FILE=$(abspath build/container/external/toolchain/gcc.$(CEPHD_PLATFORM).cmake) $(abspath ceph); \ + cd $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM) && cmake $(CEPHD_CMAKE) $(abspath ceph); \ echo "$(CEPHD_CMAKE)" > cephd.cmake; \ fi @rm $(CEPHD_BUILD_DIR)/$(CEPHD_PLATFORM)/cephd.cmake.new diff --git a/build/makelib/cross.mk b/build/makelib/cross.mk index c0340c508..4c4facd3f 100644 --- a/build/makelib/cross.mk +++ b/build/makelib/cross.mk @@ -17,24 +17,28 @@ # ifeq ($(GOOS)_$(GOARCH),linux_amd64) +CROSS_TRIPLE=x86_64-linux-gnu CC=x86_64-linux-gnu-gcc CXX=x86_64-linux-gnu-g++ STRIP=x86_64-linux-gnu-strip endif ifeq ($(GOOS)_$(GOARCH),linux_arm64) +CROSS_TRIPLE=aarch64-linux-gnu CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ STRIP=aarch64-linux-gnu-strip endif ifeq ($(GOOS)_$(GOARCH),darwin_amd64) +CROSS_TRIPLE=x86_64-apple-darwin15 CC=o64-clang CXX=o64-clang++ STRIP=x86_64-apple-darwin15-strip endif ifeq ($(CEPHD_PLATFORM),windows_amd64) +CROSS_TRIPLE=x86_64-w64-mingw32 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ STRIP=x86_64-w64-mingw32-strip diff --git a/build/run b/build/run index dc184b71b..63c8a4823 100755 --- a/build/run +++ b/build/run @@ -128,6 +128,8 @@ docker run \ -e GO_PKG_DIR="" \ -e GITHUB_TOKEN \ -e VERSION \ + -e CROSSBUILD=1 \ + -e LINKMODE=static \ ${TTY_ARGS} \ ${CCACHE_ARGS} \ ${NETRC_ARGS} \ diff --git a/ceph b/ceph index 49fdbcfdb..14a371c12 160000 --- a/ceph +++ b/ceph @@ -1 +1 @@ -Subproject commit 49fdbcfdbbe6afa6cc0447c49fa2d61465507a89 +Subproject commit 14a371c129ed9a751aa805993c3ca8e334c47a7c diff --git a/pkg/cephmgr/cephd/cephd.go b/pkg/cephmgr/cephd/cephd.go index 304ecb4a1..76941e08e 100644 --- a/pkg/cephmgr/cephd/cephd.go +++ b/pkg/cephmgr/cephd/cephd.go @@ -46,10 +46,13 @@ package cephd // #cgo tcmalloc,static LDFLAGS: -ltcmalloc_minimal // #cgo jemalloc,dynamic LDFLAGS: -Wl,-Bstatic -ljemalloc -Wl,-Bdynamic // #cgo tcmalloc,dynamic LDFLAGS: -Wl,-Bstatic -ltcmalloc_minimal -Wl,-Bdynamic +// #cgo jemalloc,stdlib LDFLAGS: -Wl,-Bstatic -ljemalloc -Wl,-Bdynamic +// #cgo tcmalloc,stdlib LDFLAGS: -Wl,-Bstatic -ltcmalloc_minimal -Wl,-Bdynamic // #cgo jemalloc tcmalloc CFLAGS: -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free // #cgo jemalloc tcmalloc CXXFLAGS: -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -// #cgo static LDFLAGS: -Wl,--whole-archive -lpthread -Wl,--no-whole-archive -lcephd -lboost_system -lboost_thread -lboost_iostreams -lboost_random -lblkid -lz -lsnappy -lcrypto++ -laio -luuid -lcurl -lfcgi -lexpat -lm -ldl -lresolv -// #cgo dynamic LDFLAGS: -Wl,-Bstatic -lcephd -lboost_system -lboost_thread -lboost_iostreams -lboost_random -lblkid -lz -lsnappy -lcrypto++ -laio -luuid -lcurl -lfcgi -lexpat -Wl,-Bdynamic -ldl -lm -lresolv +// #cgo static LDFLAGS: -Wl,--whole-archive -lpthread -Wl,--no-whole-archive -lcephd -lboost_system -lboost_thread -lboost_iostreams -lboost_random -lblkid -lz -lsnappy -lcryptopp -lrocksdb -laio -luuid -lcurl -lexpat -lm -ldl -lresolv +// #cgo stdlib LDFLAGS: -Wl,-Bstatic -lcephd -lboost_system -lboost_thread -lboost_iostreams -lboost_random -lblkid -lz -lsnappy -lcryptopp -lrocksdb -laio -luuid -lcurl -lexpat -Wl,-Bdynamic -ldl -lm -lresolv +// #cgo dynamic LDFLAGS: -Wl,-Bstatic -lcephd -Wl,-Bdynamic -lboost_system -lboost_thread -lboost_iostreams -lboost_random -lblkid -lz -lsnappy -lcryptopp -lrocksdb -laio -luuid -lcurl -lexpat -ldl -lm -lresolv // #include // #include // #include From a2d5715e07ab3d6cf6ad8e87d87a94f906c4c4c6 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Mon, 12 Dec 2016 17:20:54 -0500 Subject: [PATCH 4/4] build: fix support for jemalloc/tcmalloc jemalloc and tcmalloc are now supported as allocators --- ceph | 2 +- pkg/cephmgr/cephd/malloc.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 pkg/cephmgr/cephd/malloc.go diff --git a/ceph b/ceph index 14a371c12..3e6d1349d 160000 --- a/ceph +++ b/ceph @@ -1 +1 @@ -Subproject commit 14a371c129ed9a751aa805993c3ca8e334c47a7c +Subproject commit 3e6d1349d4ceb85247e132e920f89e29fbd760ce diff --git a/pkg/cephmgr/cephd/malloc.go b/pkg/cephmgr/cephd/malloc.go new file mode 100644 index 000000000..bbe4e416a --- /dev/null +++ b/pkg/cephmgr/cephd/malloc.go @@ -0,0 +1,36 @@ +// +build jemalloc tcmalloc + +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cephd + +// this bit of magic is used to workaround an issue +// with glibc 2.24 when replacing memory allocators. Jemalloc and tcmalloc +// both rely on providing implementations of malloc (calloc, free, etc.) +// so that the linker drops the need for glib's malloc.o and use +// jemalloc/tcmalloc's implementation instead. However, in glibc 2.24 +// fork() now calls internal functions specific to glib's memory allocator. +// The functions below provide empty implementations so that the linker +// continues to drop the need for malloc.o and pickup jemalloc/tcmalloc +// insead. See https://github.com/jemalloc/jemalloc/issues/442#event-840687583 +// and https://github.com/gperftools/gperftools/issues/856 for more context. +// Also this was fixed in glibc 2.25 but that is not widely deployed +// https://sourceware.org/bugzilla/show_bug.cgi?id=20432 + +// void __malloc_fork_lock_parent(void){}; +// void __malloc_fork_unlock_parent(void){}; +// void __malloc_fork_unlock_child(void){}; +import "C"