diff --git a/images/Makefile b/images/Makefile new file mode 100644 index 000000000..1eb4abc8d --- /dev/null +++ b/images/Makefile @@ -0,0 +1,85 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build push clean +all: build + +include image.mk + +PLATFORMS ?= amd64 arm arm64 + +.PHONY: cross-build-images +cross-build-images: + @$(MAKE) -C base GOARCH=amd64 + @$(MAKE) -C cross-build-gnu + @$(MAKE) -C cross-build-go + @$(MAKE) -C cross-build + +build.%: + @$(MAKE) -C base build GOARCH=$* + @$(MAKE) -C rookd build GOARCH=$* + +build: cross-build-images + @$(MAKE) build.$(GOARCH) + +push.%: + @$(MAKE) -C rookd push GOARCH=$* +push: $(foreach p,$(PLATFORMS), push.$(p)) + +clean: clean.build + +cross.all: $(foreach p,$(PLATFORMS), build.$(p)) +cross: cross-build-images + @$(MAKE) cross.all + +# ==================================================================================== +# Help + +.PHONY: help +help: + @echo 'Usage: make ... ' + @echo '' + @echo 'Targets:' + @echo ' build Build images for the host platform.' + @echo ' clean Remove all images created from the' + @echo ' current build.' + @echo ' cross Build images for all platforms.' + @echo ' prune Prunes orphaned and cached images at the.' + @echo ' host level.' + @echo ' push Pushes public images to registry.' + @echo '' + @echo 'Advanced Targets:' + @echo ' cross-build-images Build images used for cross-builds.' + @echo ' clean.all Remove all images including cached.' + @echo ' images at the host level.' + @echo ' prune.all Prune everything ignoring policy.' + @echo '' + @echo 'Options:' + @echo ' PLATFORMS The platforms to build when running the.' + @echo ' cross target. Default is "arm arm64 amd64"' + @echo ' PULL Whether to pull base images. Default is 1.' + @echo ' REGISTRY The docker registry to push to. Default is' + @echo ' quay.io/rook.' + @echo ' V Set to 1 enable verbose build. Default is 0.' + @echo ' VERSION The version used for image tags. The default' + @echo ' is obtained from git.' + @echo '' + @echo 'Advanced Options:' + @echo ' PRUNE_HOURS The number of hours from when an image is' + @echo ' created for it to be considered a target for' + @echo ' pruning. Default is 48 hrs.' + @echo ' PRUNE_KEEP_CACHED The minimum number of cached images to keep.' + @echo ' Default is 24 images.' + @echo ' PRUNE_KEEP_ORPHANS The minimum number of orphaned images to keep.' + @echo ' Default is 24 images.' diff --git a/images/README.md b/images/README.md new file mode 100644 index 000000000..362c7d2af --- /dev/null +++ b/images/README.md @@ -0,0 +1,71 @@ +# Images + +This folder contains all images used by the Rook project: + + - base - this is a small base Ubuntu image + +# Versioning + +Be sure to bump the version number everytime you make a change to the container, +regardless of how small that change. + +# Build Requirements + +A capable machine (2+ cores, 8+ GB of memory) with Docker installed locally: + + * MacOS: you can use Docker for Mac or your own docker-machine + * Linux: any distro with a recent version of Docker would work + +We do not currently support building on a remote docker host. + +## Cross Building Containers + +If you are using a recent Docker for Mac build you can skip this section, since +they added support for binfmt and multi-arch docker builds. + +If you are planning on cross building the arm and arm64 containers you must +install QEMU and enable binfmt support. On an Ubuntu machine with a 4.8+ kernel +you need to run install the following: + +``` +DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + binfmt-support + qemu-user-static +``` + +You also need to run the following on every boot: + +``` +docker run --rm --privileged hypriot/qemu-register +``` + +you can install a systemd unit to help with this if you'd like, for example: + +``` +cat < /etc/systemd/system/update-binfmt.service +[Unit] +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/docker run --rm --privileged hypriot/qemu-register +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +EOF + +systemctl enable update-binfmt.service +``` + +## Checking if multi-arch is supported + +To test run the following: + +``` +> docker run --rm -ti armhf/alpine uname -a +Linux 6ab4ed3ad07a 4.10.0-20-generic #22-Ubuntu SMP Thu Apr 20 09:22:42 UTC 2017 armv7l Linux + +> docker run --rm -ti aarch64/alpine uname -a +Linux 758d8063d612 4.10.0-20-generic #22-Ubuntu SMP Thu Apr 20 09:22:42 UTC 2017 aarch64 Linux +``` diff --git a/images/base/Dockerfile b/images/base/Dockerfile new file mode 100644 index 000000000..578c277fc --- /dev/null +++ b/images/base/Dockerfile @@ -0,0 +1,25 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +# APT caching to speed up apt-get downloads +# see http://www.stuartaxon.com/2015/02/02/docker-and-caching-apt-get-for-guests-on-the-same-host/ +RUN HOST_IP=$(route -n | awk '/^0.0.0.0/ {print $2}') && \ + echo "#!/bin/bash" > /usr/local/bin/apt-ng-host-discover && \ + echo "if nc -w1 -z $HOST_IP 3142; then printf http://$HOST_IP:3142; else printf DIRECT; fi" >> /usr/local/bin/apt-ng-host-discover && \ + chmod +x /usr/local/bin/apt-ng-host-discover && \ + echo 'Acquire::http::ProxyAutoDetect "/usr/local/bin/apt-ng-host-discover";' > /etc/apt/apt.conf.d/30proxy + +CMD ["/bin/bash"] diff --git a/images/base/Dockerfile.rootfs b/images/base/Dockerfile.rootfs new file mode 100644 index 000000000..0d1ecf3ea --- /dev/null +++ b/images/base/Dockerfile.rootfs @@ -0,0 +1,16 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM scratch +ADD rootfs.tar / diff --git a/images/base/Dockerfile.rootfs-build b/images/base/Dockerfile.rootfs-build new file mode 100644 index 000000000..ecb667c72 --- /dev/null +++ b/images/base/Dockerfile.rootfs-build @@ -0,0 +1,45 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +RUN sed -i -e 's/^deb-src/#deb-src/' /etc/apt/sources.list && \ + # update and upgrade + DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy --no-install-recommends \ + net-tools \ + netcat && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -yy --no-install-recommends && \ + # cleanup + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean -y + +RUN \ + # keep the copyright files + tar -czf /usr/share/copyrights.tar.xz /usr/share/doc/*/copyright && \ + # remove unecessary files + rm -rf \ + /usr/share/doc \ + /usr/share/man \ + /usr/share/info \ + /usr/share/locale \ + /var/lib/apt/lists/* \ + /var/log/* \ + /var/cache/debconf/* \ + /etc/systemd \ + /lib/lsb \ + /lib/udev \ + /usr/lib/x86_64-linux-gnu/gconv/IBM* \ + /usr/lib/x86_64-linux-gnu/gconv/EBC* && \ + mkdir -p /usr/share/man/man{1..8} diff --git a/images/base/Makefile b/images/base/Makefile new file mode 100755 index 000000000..31e626aa3 --- /dev/null +++ b/images/base/Makefile @@ -0,0 +1,65 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build clean +all: build + +include ../image.mk + +# ==================================================================================== +# Build Options + +REPO ?= base +IMAGE = $(BUILD_REGISTRY)/$(REPO)-$(GOARCH) + +# ==================================================================================== +# Build rootfs + +ROOTFS_BUILD_IMAGE := $(BUILD_REGISTRY)/rootfs-build-$(GOARCH) +ROOTFS_IMAGE := $(HOST_REGISTRY)/rootfs-$(GOARCH):$$(docker images -q $(ROOTFS_BUILD_IMAGE)) +ROOTFS_FILENAME = rootfs.tar + +TEMP := $(shell mktemp -d) +CLEAN_IMAGES += $(IMAGE) $(ROOTFS_BUILD_IMAGE) + +# NOTE: we check if docker build actually built anything (due to changes) +# by comparing the image id before and after calling docker build. If there +# were changes then we proceed with extracting the rootfs and creating the +# final image. This prevents uncessary builds of containers that use this base image. + +rootfs-build: + @echo === docker build $(ROOTFS_BUILD_IMAGE) + @cp Dockerfile.rootfs-build $(TEMP)/Dockerfile + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(OSBASEIMAGE)|g' Dockerfile + @docker build $(BUILD_BASE_ARGS) -t $(ROOTFS_BUILD_IMAGE) $(TEMP) + @[ -n "$$(docker images -q $(ROOTFS_IMAGE))" ] || $(MAKE) rootfs + +rootfs: + @echo === docker build $(ROOTFS_IMAGE) + @CID=`docker create $(ROOTFS_BUILD_IMAGE) /bin/bash` &&\ + docker export $$CID > $(TEMP)/$(ROOTFS_FILENAME) &&\ + docker rm -f $$CID + + @cp Dockerfile.rootfs $(TEMP)/Dockerfile + @docker build $(BUILD_ARGS) -t $(ROOTFS_IMAGE) $(TEMP) + @rm -fr $(TEMP) + +build: rootfs-build + @echo === docker build $(IMAGE) + @cp Dockerfile $(TEMP) + @cd $(TEMP) && $(SED_CMD) "s|BASEIMAGE|$(ROOTFS_IMAGE)|g" Dockerfile + @docker build $(BUILD_ARGS) -t $(IMAGE) $(TEMP) + @rm -fr $(TEMP) + +clean: clean.images \ No newline at end of file diff --git a/images/ceph-build/CMakeLists.txt b/images/ceph-build/CMakeLists.txt new file mode 100644 index 000000000..657f3ff49 --- /dev/null +++ b/images/ceph-build/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 2.8.11) + +# make all install targets OPTIONAL by default by overriding the install function +function(install) + _install(${ARGN} OPTIONAL) +endfunction() + +include(CMakeLists.original.txt) + +# the ceph targets we need for rook. be careful bringing +# in new targets as the container might mushroom in size +add_custom_target(rook DEPENDS + ceph-authtool + ceph-common + ceph-mds + ceph-mgr + ceph-mon + ceph-osd + cephfs + rados + radosgw + radosgw-admin + rbd + rgw +) diff --git a/images/ceph-build/Dockerfile b/images/ceph-build/Dockerfile new file mode 100644 index 000000000..e3f191f18 --- /dev/null +++ b/images/ceph-build/Dockerfile @@ -0,0 +1,106 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +# clone the repo. we do this first so that this layer can be reused across archs. +ARG CEPH_GIT_REPO +ARG CEPH_GIT_BRANCH +RUN git clone -b ${CEPH_GIT_BRANCH} --recurse-submodules ${CEPH_GIT_REPO} ceph + +# install libraries ceph needs during build +ARG ARCH +ARG CROSS_TRIPLE +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy -q --no-install-recommends \ + python-setuptools \ + \ + cython:${ARCH} \ + python:${ARCH} \ + libaio-dev:${ARCH} \ + libatomic-ops-dev:${ARCH} \ + libbabeltrace-dev:${ARCH} \ + libblkid-dev:${ARCH} \ + libboost-context-dev:${ARCH} \ + libboost-coroutine-dev:${ARCH} \ + libboost-date-time-dev:${ARCH} \ + libboost-iostreams-dev:${ARCH} \ + libboost-program-options-dev:${ARCH} \ + libboost-python-dev:${ARCH} \ + libboost-random-dev:${ARCH} \ + libboost-regex-dev:${ARCH} \ + libboost-system-dev:${ARCH} \ + libboost-thread-dev:${ARCH} \ + libcurl4-gnutls-dev:${ARCH} \ + libexpat1-dev:${ARCH} \ + libgoogle-perftools-dev:${ARCH} \ + libgoogle-perftools4:${ARCH} \ + libibverbs-dev:${ARCH} \ + libjemalloc-dev:${ARCH} \ + libkeyutils-dev:${ARCH} \ + libldap2-dev:${ARCH} \ + libnss3-dev:${ARCH} \ + libpython-dev:${ARCH} \ + libsnappy-dev:${ARCH} \ + libssl-dev:${ARCH} \ + libtcmalloc-minimal4:${ARCH} \ + libudev-dev:${ARCH} \ + libunwind-dev:${ARCH} \ + zlib1g-dev:${ARCH} && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# checkout the commit hash and update submodules +ARG CEPH_GIT_COMMIT +RUN cd /build/ceph && \ + git fetch --all --prune && \ + git checkout -b ceph-builder ${CEPH_GIT_COMMIT} && \ + git submodule update --init --recursive + +# create a wrapper CMakelists.txt to enable partial building +# and installation of targets. We dont want to build all of +# ceph -- just a subset of targets that we need. +# see http://stackoverflow.com/questions/17164731/installing-only-one-target-and-its-dependencies-out-of-a-complex-project-with +RUN mv /build/ceph/CMakeLists.txt /build/ceph/CMakeLists.original.txt +ADD CMakeLists.txt /build/ceph + +# configure ceph +ARG CEPH_BUILD_TYPE +ARG CEPH_ALLOCATOR +RUN mkdir -p /build/ceph/build && cd /build/ceph/build && \ + cmake \ + -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=${CEPH_BUILD_TYPE} \ + -DCMAKE_TOOLCHAIN_FILE=/usr/local/toolchain/${CROSS_TRIPLE}.cmake \ + -DALLOCATOR=${CEPH_ALLOCATOR} \ + -DWITH_SYSTEM_BOOST=ON \ + -DWITH_EMBEDDED=OFF \ + -DWITH_FUSE=OFF \ + -DWITH_LEVELDB=OFF \ + -DWITH_LTTNG=OFF \ + -DWITH_MANPAGE=OFF \ + -DWITH_PROFILER=OFF \ + -DWITH_PYTHON3=OFF \ + -DWITH_RADOSGW_FCGI_FRONTEND=OFF \ + .. + +WORKDIR /build/ceph/build + +# now do the actual building. Note we dont build inside the Dockerfile +# since we are unable to bind a volume and use ccache +ADD build-ceph.sh /build/ +ENTRYPOINT ["/build/build-ceph.sh"] diff --git a/images/ceph-build/Makefile b/images/ceph-build/Makefile new file mode 100644 index 000000000..0c1ebf795 --- /dev/null +++ b/images/ceph-build/Makefile @@ -0,0 +1,89 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build push +all: build + +include ../image.mk + +# ==================================================================================== +# Options + +# the git repo to build from +CEPH_GIT_REPO ?= https://github.com/rook/ceph + +CEPH_GIT_BRANCH ?= rook-luminous + +# the git commit hash to build +ifeq ($(origin CEPH_GIT_COMMIT), undefined) +CEPH_GIT_COMMIT := $(shell cat commit_hash) +endif + +# set to release or relwithdebinfo (make sure its lower case) +CEPH_BUILD_TYPE ?= release + +# set to tcmalloc, tcmalloc_minimal, jemalloc or libc +CEPH_ALLOCATOR ?= tcmalloc +ifeq ($(filter libc tcmalloc tcmalloc_minimal jemalloc, $(CEPH_ALLOCATOR)),) +$(error unsupported memory allocator $(CEPH_ALLOCATOR)) +endif + +CEPH_CONFIG ?= $(GOARCH)-$(CEPH_ALLOCATOR)-$(CEPH_BUILD_TYPE) +BASEIMAGE := $(BUILD_REGISTRY)/cross-build-gnu +CEPH_BUILD_IMAGE := $(BUILD_REGISTRY)/ceph-build-$(CEPH_CONFIG) +CEPH_IMAGE := $(HOST_REGISTRY)/ceph-$(CEPH_CONFIG):$$(docker images -q $(CEPH_BUILD_IMAGE)) +CLEAN_IMAGES = $(CEPH_BUILD_IMAGE) + +CEPH_TARBALL = ceph-install.tar + +TEMP := $(shell mktemp -d) + +# ==================================================================================== +# Targets + +build: + @echo === docker build $(CEPH_BUILD_IMAGE) + @cp -a . $(TEMP) + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) \ + --build-arg ARCH=$(DEBIAN_ARCH) \ + --build-arg CEPH_GIT_COMMIT=$(CEPH_GIT_COMMIT) \ + --build-arg CEPH_GIT_REPO=$(CEPH_GIT_REPO) \ + --build-arg CEPH_GIT_BRANCH=$(CEPH_GIT_BRANCH) \ + --build-arg CEPH_ALLOCATOR=$(CEPH_ALLOCATOR) \ + --build-arg CEPH_BUILD_TYPE=$(CEPH_BUILD_TYPE) \ + --build-arg CROSS_TRIPLE=$(CROSS_TRIPLE) \ + -t $(CEPH_BUILD_IMAGE) \ + $(TEMP) + @[ -n "$$(docker images -q $(CEPH_IMAGE))" ] || $(MAKE) ceph + +ceph: + @echo === building ceph from $(CEPH_BUILD_IMAGE) + @docker run \ + -i $(RUN_ARGS) \ + --cidfile $(TEMP)/cid \ + -e CCACHE_DIR=/home/build/.ccache \ + -v $(HOME)/.ccache:/home/build/.ccache \ + $(CEPH_BUILD_IMAGE) + @docker cp `cat $(TEMP)/cid`:/build/$(CEPH_TARBALL) $(TEMP)/$(CEPH_TARBALL) + @docker rm -f `cat $(TEMP)/cid` + @docker import $(TEMP)/$(CEPH_TARBALL) $(CEPH_IMAGE) + @rm -fr $(TEMP) + +%/$(CEPH_TARBALL): build + @CID=`docker create $(CEPH_IMAGE) /dev/null` &&\ + docker export $$CID > $@ &&\ + docker rm -f $$CID + +clean: clean.images \ No newline at end of file diff --git a/images/ceph-build/build-ceph.sh b/images/ceph-build/build-ceph.sh new file mode 100755 index 000000000..d962654a7 --- /dev/null +++ b/images/ceph-build/build-ceph.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CEPH_INSTALL_DIR=${CEPH_INSTALL_DIR:-/build/ceph-install} +CEPH_TARBALL=${CEPH_INSTALL_DIR}.tar + +makeargs=-j$(( $(nproc) - 1 )) + +# build and install +make ${makeargs} rook +make ${makeargs} DESTDIR="${CEPH_INSTALL_DIR}" install + +# create the ceph tar file +tar --numeric-owner --create --file "${CEPH_TARBALL}" --directory "${CEPH_INSTALL_DIR}" --transform='s,^./,,' . diff --git a/images/ceph-build/commit_hash b/images/ceph-build/commit_hash new file mode 100644 index 000000000..a6b7c0a5f --- /dev/null +++ b/images/ceph-build/commit_hash @@ -0,0 +1 @@ +71677375ca98b267e07d99239194ba9f9c4b1889 \ No newline at end of file diff --git a/images/cross-build-gnu/Dockerfile b/images/cross-build-gnu/Dockerfile new file mode 100644 index 000000000..45c06401d --- /dev/null +++ b/images/cross-build-gnu/Dockerfile @@ -0,0 +1,53 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +# install build tools, compilers and cross compilers for all supported platforms +RUN echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu zesty main universe restricted" > /etc/apt/sources.list &&\ + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu zesty-updates main universe restricted" >> /etc/apt/sources.list &&\ + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu zesty-security main universe restricted" >> /etc/apt/sources.list &&\ + echo "deb [arch=amd64] http://archive.ubuntu.com/ubuntu zesty-backports main universe restricted" >> /etc/apt/sources.list &&\ + echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ zesty main universe restricted" >> /etc/apt/sources.list &&\ + echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ zesty-updates main universe restricted" >> /etc/apt/sources.list &&\ + echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ zesty-security main universe restricted" >> /etc/apt/sources.list &&\ + echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ zesty-backports main universe restricted" >> /etc/apt/sources.list &&\ + dpkg --add-architecture armhf && \ + dpkg --add-architecture arm64 && \ + DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy -q --no-install-recommends \ + build-essential \ + ca-certificates \ + ccache \ + cmake \ + crossbuild-essential-arm64 \ + crossbuild-essential-armhf \ + curl \ + git \ + jq \ + patch \ + yasm \ + zip && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# add cmake toolchains +ADD toolchain /usr/local/toolchain + +# set the working directory +WORKDIR /build + +ENV TERM=xterm diff --git a/images/cross-build-gnu/Makefile b/images/cross-build-gnu/Makefile new file mode 100644 index 000000000..7dd6fde17 --- /dev/null +++ b/images/cross-build-gnu/Makefile @@ -0,0 +1,42 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build clean +all: build + +# cross-build only supported on amd64 +override GOARCH := amd64 +include ../image.mk + +# ==================================================================================== +# Build Options + +REPO ?= cross-build-gnu +IMAGE = $(BUILD_REGISTRY)/$(REPO) +BASEIMAGE = $(BUILD_REGISTRY)/base-$(GOARCH) + +# ==================================================================================== +# Build image + +TEMP := $(shell mktemp -d) +CLEAN_IMAGES := $(IMAGE) + +build: + @echo === docker build $(IMAGE) + @cp -a . $(TEMP) + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) -t $(IMAGE) $(TEMP) + @rm -fr $(TEMP) + +clean: clean.images \ No newline at end of file diff --git a/images/cross-build-gnu/toolchain/aarch64-linux-gnu.cmake b/images/cross-build-gnu/toolchain/aarch64-linux-gnu.cmake new file mode 100644 index 000000000..68b5e7be2 --- /dev/null +++ b/images/cross-build-gnu/toolchain/aarch64-linux-gnu.cmake @@ -0,0 +1,37 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +set(cross_triple "aarch64-linux-gnu") + +set(CMAKE_C_COMPILER "/usr/lib/ccache/${cross_triple}-gcc" CACHE PATH "C compiler") +set(CMAKE_CXX_COMPILER "/usr/lib/ccache/${cross_triple}-g++" CACHE PATH "C++ compiler") +set(CMAKE_ASM_COMPILER "/usr/bin/${cross_triple}-gcc" CACHE PATH "assembler") +set(CMAKE_STRIP "/usr/bin/${cross_triple}-strip" CACHE PATH "strip") +set(CMAKE_AR "/usr/bin/${cross_triple}-ar" CACHE PATH "archive") +set(CMAKE_LINKER "/usr/bin/${cross_triple}-ld" CACHE PATH "linker") +set(CMAKE_NM "/usr/bin/${cross_triple}-nm" CACHE PATH "nm") +set(CMAKE_OBJCOPY "/usr/bin/${cross_triple}-objcopy" CACHE PATH "objcopy") +set(CMAKE_OBJDUMP "/usr/bin/${cross_triple}-objdump" CACHE PATH "objdump") +set(CMAKE_RANLIB "/usr/bin/${cross_triple}-ranlib" CACHE PATH "ranlib") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/qemu-aarch64-static) diff --git a/images/cross-build-gnu/toolchain/arm-linux-gnueabihf.cmake b/images/cross-build-gnu/toolchain/arm-linux-gnueabihf.cmake new file mode 100644 index 000000000..cc7fa8366 --- /dev/null +++ b/images/cross-build-gnu/toolchain/arm-linux-gnueabihf.cmake @@ -0,0 +1,37 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR armhf) + +set(cross_triple "arm-linux-gnueabihf") + +set(CMAKE_C_COMPILER "/usr/lib/ccache/${cross_triple}-gcc" CACHE PATH "C compiler") +set(CMAKE_CXX_COMPILER "/usr/lib/ccache/${cross_triple}-g++" CACHE PATH "C++ compiler") +set(CMAKE_ASM_COMPILER "/usr/bin/${cross_triple}-gcc" CACHE PATH "assembler") +set(CMAKE_STRIP "/usr/bin/${cross_triple}-strip" CACHE PATH "strip") +set(CMAKE_AR "/usr/bin/${cross_triple}-ar" CACHE PATH "archive") +set(CMAKE_LINKER "/usr/bin/${cross_triple}-ld" CACHE PATH "linker") +set(CMAKE_NM "/usr/bin/${cross_triple}-nm" CACHE PATH "nm") +set(CMAKE_OBJCOPY "/usr/bin/${cross_triple}-objcopy" CACHE PATH "objcopy") +set(CMAKE_OBJDUMP "/usr/bin/${cross_triple}-objdump" CACHE PATH "objdump") +set(CMAKE_RANLIB "/usr/bin/${cross_triple}-ranlib" CACHE PATH "ranlib") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/qemu-arm-static) diff --git a/images/cross-build-gnu/toolchain/x86_64-linux-gnu.cmake b/images/cross-build-gnu/toolchain/x86_64-linux-gnu.cmake new file mode 100644 index 000000000..ac589c8d9 --- /dev/null +++ b/images/cross-build-gnu/toolchain/x86_64-linux-gnu.cmake @@ -0,0 +1,37 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR x86_64) + +set(cross_triple "x86_64-linux-gnu") + +set(CMAKE_C_COMPILER "/usr/lib/ccache/${cross_triple}-gcc" CACHE PATH "C compiler") +set(CMAKE_CXX_COMPILER "/usr/lib/ccache/${cross_triple}-g++" CACHE PATH "C++ compiler") +set(CMAKE_ASM_COMPILER "/usr/bin/${cross_triple}-gcc" CACHE PATH "assembler") +set(CMAKE_STRIP "/usr/bin/${cross_triple}-strip" CACHE PATH "strip") +set(CMAKE_AR "/usr/bin/${cross_triple}-ar" CACHE PATH "archive") +set(CMAKE_LINKER "/usr/bin/${cross_triple}-ld" CACHE PATH "linker") +set(CMAKE_NM "/usr/bin/${cross_triple}-nm" CACHE PATH "nm") +set(CMAKE_OBJCOPY "/usr/bin/${cross_triple}-objcopy" CACHE PATH "objcopy") +set(CMAKE_OBJDUMP "/usr/bin/${cross_triple}-objdump" CACHE PATH "objdump") +set(CMAKE_RANLIB "/usr/bin/${cross_triple}-ranlib" CACHE PATH "ranlib") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/qemu-x86_64-static) diff --git a/images/cross-build-go/Dockerfile b/images/cross-build-go/Dockerfile new file mode 100644 index 000000000..1e71f9ed3 --- /dev/null +++ b/images/cross-build-go/Dockerfile @@ -0,0 +1,67 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +# install the OSX and Win32 cross compilers. see https://github.com/tpoechtrager/osxcross +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy -q --no-install-recommends \ + clang \ + gcc-mingw-w64 \ + libxml2-dev \ + llvm-dev && \ + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN OSXCROSS_COMMIT_HASH=474f359d2f27ff68916a064f0138c9188c63db7d && \ + OSXCROSS_SDK=MacOSX10.11.sdk && \ + OSXCROSS_SDK_HASH=d75dddc8d820daf38e90a7c8e9771750a8a4371027a2d658a904cffacbf6b09b && \ + git clone https://github.com/tpoechtrager/osxcross.git && \ + git -C osxcross checkout -b rook-build ${OSXCROSS_COMMIT_HASH} && \ + curl -fsSL https://castle-cross-build.s3.amazonaws.com/osxcross/${OSXCROSS_SDK}.tar.xz -o osxcross/tarballs/${OSXCROSS_SDK}.tar.xz && \ + echo "${OSXCROSS_SDK_HASH} osxcross/tarballs/${OSXCROSS_SDK}.tar.xz" | sha256sum -c - && \ + UNATTENDED=yes OSX_VERSION_MIN=10.6 JOBS=4 osxcross/build.sh && \ + mv osxcross/target /usr/local/osxcross && \ + rm -rf osxcross && \ + rm -rf /usr/local/osxcross/SDK/${OSXCROSS_SDK}/usr/share +ENV PATH /usr/local/osxcross/bin:$PATH + +# install go-lang +RUN GO_VERSION=1.8.3 && \ + GO_HASH=1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 && \ + curl -fsSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz -o golang.tar.gz && \ + echo "${GO_HASH} golang.tar.gz" | sha256sum -c - && \ + tar -C /usr/local -xzf golang.tar.gz && \ + rm golang.tar.gz +ENV PATH /usr/local/go/bin:$PATH + +# precompile the go standard library for all supported platforms and configurations +# the install suffixes match those in golang.mk so please keep them in sync +RUN export GOARM=7 && \ + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 CC=x86_64-apple-darwin15-clang go install -a std && \ + CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 CC=x86_64-apple-darwin15-clang go install -installsuffix static -a std && \ + CGO_ENABLED=1 GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc go install -a std && \ + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 CC=x86_64-w64-mingw32-gcc go install -installsuffix static -a std && \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm CC=arm-linux-gnueabihf-gcc go install -a std && \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm CC=arm-linux-gnueabihf-gcc go install -buildmode pie -installsuffix pie -a std && \ + CGO_ENABLED=0 GOOS=linux GOARCH=arm CC=arm-linux-gnueabihf-gcc go install -installsuffix static -a std && \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc go install -a std && \ + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc go install -buildmode pie -installsuffix pie -a std && \ + CGO_ENABLED=0 GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc go install -installsuffix static -a std && \ + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-gnu-gcc go install -a std && \ + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-gnu-gcc go install -buildmode pie -installsuffix pie -a std && \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 CC=x86_64-linux-gnu-gcc go install -installsuffix static -a std && \ + \ + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-gnu-gcc go install -installsuffix race -a std + diff --git a/images/cross-build-go/Makefile b/images/cross-build-go/Makefile new file mode 100644 index 000000000..faaf2a303 --- /dev/null +++ b/images/cross-build-go/Makefile @@ -0,0 +1,42 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build clean +all: build + +# cross-build only supported on amd64 +override GOARCH := amd64 +include ../image.mk + +# ==================================================================================== +# Build Options + +REPO ?= cross-build-go +IMAGE = $(BUILD_REGISTRY)/$(REPO) +BASEIMAGE = $(BUILD_REGISTRY)/cross-build-gnu + +# ==================================================================================== +# Build image + +TEMP := $(shell mktemp -d) +CLEAN_IMAGES := $(IMAGE) + +build: + @echo === docker build $(IMAGE) + @cp -a . $(TEMP) + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) -t $(IMAGE) $(TEMP) + @rm -fr $(TEMP) + +clean: clean.images \ No newline at end of file diff --git a/images/cross-build/Dockerfile b/images/cross-build/Dockerfile new file mode 100644 index 000000000..be99d7d81 --- /dev/null +++ b/images/cross-build/Dockerfile @@ -0,0 +1,43 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +# install build tools, compilers and cross compilers for all supported platforms +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy -q --no-install-recommends \ + awscli \ + curl \ + docker.io \ + lsb-release \ + rsync \ + runit \ + sudo \ + xz-utils && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# install acbuild and docker2aci +RUN ACBUILD_VERSION=0.4.0 && \ + curl -fsSLO https://github.com/containers/build/releases/download/v${ACBUILD_VERSION}/acbuild-v${ACBUILD_VERSION}.tar.gz && \ + tar -C /usr/local/bin --strip-components=1 -xzf acbuild-v${ACBUILD_VERSION}.tar.gz && \ + rm -f acbuild-v${ACBUILD_VERSION}.tar.gz && \ + DOCKER2ACI_VERSION=0.14.0 && \ + curl -fsSLO https://github.com/appc/docker2aci/releases/download/v${DOCKER2ACI_VERSION}/docker2aci-v${DOCKER2ACI_VERSION}.tar.gz && \ + tar -C /usr/local/bin --strip-components=1 -xzf docker2aci-v${DOCKER2ACI_VERSION}.tar.gz && \ + rm -f docker2aci-v${DOCKER2ACI_VERSION}.tar.gz + +COPY entrypoint.sh rsyncd.sh /build/ +ENTRYPOINT [ "/build/entrypoint.sh" ] diff --git a/images/cross-build/Makefile b/images/cross-build/Makefile new file mode 100644 index 000000000..f96bba9d9 --- /dev/null +++ b/images/cross-build/Makefile @@ -0,0 +1,42 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build clean +all: build + +# cross-build only supported on amd64 +override GOARCH := amd64 +include ../image.mk + +# ==================================================================================== +# Build Options + +REPO ?= cross-build +IMAGE = $(BUILD_REGISTRY)/$(REPO) +BASEIMAGE = $(BUILD_REGISTRY)/cross-build-go + +# ==================================================================================== +# Build image + +TEMP := $(shell mktemp -d) +CLEAN_IMAGES := $(IMAGE) + +build: + @echo === docker build $(IMAGE) + @cp -a . $(TEMP) + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) -t $(IMAGE) $(TEMP) + @rm -fr $(TEMP) + +clean: clean.images \ No newline at end of file diff --git a/images/cross-build/README.md b/images/cross-build/README.md new file mode 100644 index 000000000..449df939a --- /dev/null +++ b/images/cross-build/README.md @@ -0,0 +1,19 @@ +# Cross Build Image + +This directory builds a container image that can be used for cross building, and +also to minimize dependencies on the host platform. + +## Building + +To build the container run make in the directory. + +## Pushing + +To publish to docker hub run `make push`. + +## Version + +Be sure to update the version number in the `version` file when making changes +to the build container. We use semantic versioning for the build container. + + diff --git a/images/cross-build/entrypoint.sh b/images/cross-build/entrypoint.sh new file mode 100755 index 000000000..4a715dfc2 --- /dev/null +++ b/images/cross-build/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e + +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARGS="$@" +if [ $# -eq 0 ]; then + ARGS=/bin/bash +fi + +BUILDER_USER=${BUILDER_USER:-rook} +BUILDER_GROUP=${BUILDER_GROUP:-rook} +BUILDER_UID=${BUILDER_UID:-1000} +BUILDER_GID=${BUILDER_GID:-1000} + +groupadd -o -g $BUILDER_GID $BUILDER_GROUP 2> /dev/null +useradd -o -m -g $BUILDER_GID -u $BUILDER_UID $BUILDER_USER 2> /dev/null +echo "$BUILDER_USER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +export HOME=/home/${BUILDER_USER} +echo "127.0.0.1 $(cat /etc/hostname)" >> /etc/hosts +[[ -S /var/run/docker.sock ]] && chown $BUILDER_UID:$BUILDER_GID /var/run/docker.sock +chown -R $BUILDER_UID:$BUILDER_GID $HOME +exec chpst -u :$BUILDER_UID:$BUILDER_GID ${ARGS} diff --git a/images/cross-build/rsyncd.sh b/images/cross-build/rsyncd.sh new file mode 100755 index 000000000..19b6a0f02 --- /dev/null +++ b/images/cross-build/rsyncd.sh @@ -0,0 +1,51 @@ +#!/bin/bash -e + +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VOLUME=${VOLUME:-/volume} +ALLOW=${ALLOW:-192.168.0.0/16 172.16.0.0/12 10.0.0.0/8} +OWNER=${OWNER:-nobody} +GROUP=${GROUP:-nogroup} + +if [[ "${GROUP}" != "nogroup" && "${GROUP}" != "root" ]]; then + groupadd -g ${GROUP} rsync +fi + +if [[ "${OWNER}" != "nobody" && "${OWNER}" != "root" ]]; then + groupadd -u ${OWNER} -G rsync rsync +fi + +chown "${OWNER}:${GROUP}" "${VOLUME}" + +[ -f /etc/rsyncd.conf ] || cat < /etc/rsyncd.conf +uid = ${OWNER} +gid = ${GROUP} +use chroot = yes +log file = /dev/stdout +reverse lookup = no +[volume] + hosts deny = * + hosts allow = ${ALLOW} + read only = false + path = ${VOLUME} + comment = volume +EOF + +for dir in ${MKDIRS}; do + mkdir -p ${dir} + chown "${OWNER}:${GROUP}" ${dir} +done + +exec /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf "$@" diff --git a/images/image.mk b/images/image.mk new file mode 100644 index 000000000..d531f8430 --- /dev/null +++ b/images/image.mk @@ -0,0 +1,151 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) + +include $(SELF_DIR)/../build/makelib/cross.mk + +# a registry that is scoped to the local host +HOSTNAME := $(shell hostname) +HOST_REGISTRY := host-$(shell echo $(HOSTNAME) | sha256sum | cut -c1-8) + +# a registry that is scoped to the current build tree on this host +ROOTDIR=$(shell cd $(SELF_DIR)/.. && pwd -P) +BUILD_REGISTRY := build-$(shell echo $(HOSTNAME)-$(ROOTDIR) | sha256sum | cut -c1-8) + +# public registry used for images that are pushed +REGISTRY ?= quay.io/rook + +# version to use if not defined +ifeq ($(origin VERSION), undefined) +VERSION := $(shell git describe --dirty --always --tags) +endif + +# the base ubuntu image to use +OSBASE ?= ubuntu:zesty + +ifeq ($(GOARCH),amd64) +OSBASEIMAGE=$(OSBASE) +endif +ifeq ($(GOARCH),arm) +OSBASEIMAGE=armhf/$(OSBASE) +endif +ifeq ($(GOARCH),arm64) +OSBASEIMAGE=aarch64/$(OSBASE) +endif + +UNAME_S:=$(shell uname -s) +ifeq ($(UNAME_S),Darwin) +SED_CMD?=sed -i "" +endif +ifeq ($(UNAME_S),Linux) +SED_CMD?=sed -i +endif + +INTERACTIVE:=$(shell [ -t 0 ] && echo 1) +ifdef INTERACTIVE +RUN_ARGS ?= -t +endif + +CACHEBUST ?= 0 +ifeq ($(CACHEBUST),1) +BUILD_ARGS += --no-cache +endif + +V ?= 0 +ifeq ($(V),1) +MAKEFLAGS += VERBOSE=1 +else +MAKEFLAGS += --no-print-directory +BUILD_ARGS ?= -q +endif + +PULL ?= 1 +ifeq ($(PULL),1) +BUILD_BASE_ARGS += --pull +endif +export PULL + +BUILD_BASE_ARGS += $(BUILD_ARGS) + +# prune images that are at least this many hours old +PRUNE_HOURS ?= 48 + +# prune keeps at least this many images regardless of how old they are +PRUNE_KEEP_CACHED ?= 24 +PRUNE_KEEP_ORPHANS ?= 24 + +CLEANUP_DATE := $(shell date -u --date="$(PRUNE_HOURS) hours ago" +"%Y-%m-%d %T %z UTC") + +# NOTE: docker rmi removes the last image when untagging. instead we switch the tag +# to a small dummy orphaning the original image and then call docker rmi. This approach +# avoids removing the last layer (and having to rebuild it) + +DUMMY_IMAGE_BASE ?= tianon/true +DUMMY_IMAGE ?= $(HOST_REGISTRY)/dummy:latest + +clean.init: + @if [ -z "`docker images -q $(DUMMY_IMAGE)`" ]; then \ + docker pull $(DUMMY_IMAGE_BASE) > /dev/null 2>&1; \ + docker tag $(DUMMY_IMAGE_BASE) $(DUMMY_IMAGE) > /dev/null 2>&1; \ + docker rmi $(DUMMY_IMAGE_BASE) > /dev/null 2>&1; \ + fi + +do.orphan: clean.init + @for i in $(CLEAN_IMAGES); do \ + [ "$$i" != "$(DUMMY_IMAGE)" ] || continue; \ + for c in $$(docker ps -a -q --filter=ancestor=$$i); do \ + echo stopping and removing container $${c} referencing image $$i; \ + docker stop $${c}; \ + docker rm $${c}; \ + done; \ + echo orphaning image $$i; \ + docker tag $(DUMMY_IMAGE) $$i > /dev/null 2>&1; \ + docker rmi $$i > /dev/null 2>&1; \ + done + +# this will clean all images in the variable CLEAN_IMAGES +clean.images: do.clean + +# this will clean everything for this build +clean.build: clean.init + @$(MAKE) do.orphan CLEAN_IMAGES="$(shell docker images | grep -E '^$(BUILD_REGISTRY)|$(REGISTRY)/' | awk '{print $$1":"$$2}')" + +# this will clean everything for this host regardless of prune policy +clean.all: clean.init + @$(MAKE) do.orphan CLEAN_IMAGES="$(shell docker images | grep -E '^$(BUILD_REGISTRY)|$(HOST_REGISTRY)|$(REGISTRY)/' | awk '{print $$1":"$$2}')" + +# prune removes old orphaned images and old cached images +prune: + @echo === pruning images older than $(PRUNE_HOURS) hours + @echo === keeping a minimum of $(PRUNE_KEEP_ORPHANS) orphaned images and $(PRUNE_KEEP_CACHED) cached images + @CLEAN_IMAGES="$(shell docker images --format "{{.CreatedAt}}#{{.Repository}}:{{.Tag}}" \ + | grep -E '$(HOST_REGISTRY)/' \ + | grep -v '$(DUMMY_IMAGE)' \ + | sort -r \ + | awk -v i=0 -v cd="$(CLEANUP_DATE)" -F "#" '{if ($$1 <= cd && i >= $(PRUNE_KEEP_CACHED)) print $$2; i++ }')" &&\ + $(MAKE) do.orphan CLEAN_IMAGES="$${CLEAN_IMAGES}" + @CLEAN_IMAGES="$(shell docker images -q -f dangling=true --format "{{.CreatedAt}}#{{.ID}}" \ + | sort -r \ + | awk -v i=0 -v cd="$(CLEANUP_DATE)" -F "#" '{if ($$1 <= cd && i >= $(PRUNE_KEEP_ORPHANS)) print $$2; i++ }')" &&\ + for i in $${CLEAN_IMAGES}; do \ + echo removing orphaned image $$i; \ + docker rmi $$i > /dev/null 2>&1; \ + done + +prune.all: + @for i in $$(docker images -q -f dangling=true); do \ + echo removing orphaned image $$i; \ + docker rmi $$i > /dev/null 2>&1; \ + done diff --git a/images/rookd/Dockerfile b/images/rookd/Dockerfile new file mode 100644 index 000000000..78ffdfd93 --- /dev/null +++ b/images/rookd/Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +ARG EXTRALIBS + +# install runtime dependencies for ceph and rookd +RUN BOOST_VERSION=1.62.0 && \ + DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yy -q --no-install-recommends \ + ca-certificates \ + libaio1 \ + libboost-context${BOOST_VERSION} \ + libboost-coroutine${BOOST_VERSION} \ + libboost-date-time${BOOST_VERSION} \ + libboost-iostreams${BOOST_VERSION} \ + libboost-program-options${BOOST_VERSION} \ + libboost-python${BOOST_VERSION} \ + libboost-random${BOOST_VERSION} \ + libboost-regex${BOOST_VERSION} \ + libboost-system${BOOST_VERSION} \ + libboost-thread${BOOST_VERSION} \ + libcurl3-gnutls \ + libexpat1 \ + libibverbs1 \ + libldap-2.4-2 \ + libnss3 \ + libpython2.7 \ + python-prettytable \ + python2.7-minimal \ + ${EXTRALIBS} && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# add the ceph binaries +ADD ceph-install.tar / + +# ensure dynamic libraries are updated +RUN ldconfig + +ENV PYTHONPATH=$PYTHONPATH:/usr/local/lib/python2.7/site-packages + +ADD rookd /usr/local/bin/ + +ENTRYPOINT ["/usr/local/bin/rookd"] diff --git a/images/rookd/Makefile b/images/rookd/Makefile new file mode 100755 index 000000000..c16eac71a --- /dev/null +++ b/images/rookd/Makefile @@ -0,0 +1,86 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all build push +all: build + +include ../image.mk + +# ==================================================================================== +# Image Build Options + +REPO ?= rookd +IMAGE = $(REGISTRY)/$(REPO)-$(GOARCH):$(VERSION) +CLEAN_IMAGES = $(IMAGE) + +# ==================================================================================== +# Ceph Build Options + +CEPH_BUILD_TYPE ?= release +CEPH_ALLOCATOR ?= tcmalloc + +# ==================================================================================== +# Common Build Settings + +BASEIMAGE = $(BUILD_REGISTRY)/base-$(GOARCH) + +TEMP := $(shell mktemp -d) + +# ==================================================================================== +# Build Ceph + +ifeq ($(CEPH_ALLOCATOR),tcmalloc) +EXTRA_LIBS += libgoogle-perftools4 +endif +ifeq ($(CEPH_ALLOCATOR),tcmalloc_minimal) +EXTRA_LIBS += libtcmalloc_minimal4 +endif +ifeq ($(CEPH_ALLOCATOR),jemalloc) +EXTRA_LIBS += libjemalloc1 +endif + +CEPH_TARBALL_FILENAME = ceph-install.tar +CEPH_TARBALL = $(TEMP)/$(CEPH_TARBALL_FILENAME) + +$(CEPH_TARBALL): + @echo === building ceph for $(IMAGE) + @$(MAKE) -C ../ceph-build GOARCH=$(GOARCH) CEPH_ALLOCATOR=$(CEPH_ALLOCATOR) CEPH_BUILD_TYPE=$(CEPH_BUILD_TYPE) $(CEPH_TARBALL) + +# ==================================================================================== +# Build Rook + +ROOKD = $(TEMP)/rookd + +$(ROOKD): + @echo === building rookd for $(IMAGE) + @../../build/run make WORKDIR=/build/.workdir GO_PKG_DIR= GOOS=linux GOARCH=$(GOARCH) + @cp ../../bin/linux_$(GOARCH)/rookd $@ + +# ==================================================================================== +# Build Final Image + +EXISTING_IMAGE_ID := $(shell docker images -q $(IMAGE)) + +build: $(CEPH_TARBALL) $(ROOKD) + @echo === docker build $(IMAGE) + @cp Dockerfile $(TEMP) + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) --build-arg EXTRALIBS=$(EXTRA_LIBS) -t $(IMAGE) $(TEMP) + @docker rmi $(EXISTING_IMAGE_ID) > /dev/null 2>&1 || true + @rm -fr $(TEMP) + +push: + @docker push $(IMAGE) + +clean: clean.images \ No newline at end of file