-
Notifications
You must be signed in to change notification settings - Fork 813
/
Dockerfile.citus
201 lines (186 loc) · 10.6 KB
/
Dockerfile.citus
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
## This Dockerfile is meant to aid in the building and debugging patroni whilst developing on your local machine
## It has all the necessary components to play/debug with a single node appliance, running etcd
ARG PG_MAJOR=16
ARG COMPRESS=false
ARG PGHOME=/home/postgres
ARG PGDATA=$PGHOME/data
ARG LC_ALL=C.UTF-8
ARG LANG=C.UTF-8
FROM postgres:$PG_MAJOR as builder
ARG PGHOME
ARG PGDATA
ARG LC_ALL
ARG LANG
ENV ETCDVERSION=3.3.13 CONFDVERSION=0.16.0
RUN set -ex \
&& export DEBIAN_FRONTEND=noninteractive \
&& echo 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";' > /etc/apt/apt.conf.d/01norecommend \
&& apt-get update -y \
# postgres:PG_MAJOR is based on debian, which has the patroni package. We will install all required dependencies
&& apt-cache depends patroni | sed -n -e 's/.*Depends: \(python3-.\+\)$/\1/p' \
| grep -Ev '^python3-(sphinx|etcd|consul|kazoo|kubernetes)' \
| xargs apt-get install -y vim curl less jq locales haproxy sudo \
python3-etcd python3-kazoo python3-pip busybox \
net-tools iputils-ping lsb-release dumb-init --fix-missing \
&& if [ $(dpkg --print-architecture) = 'arm64' ]; then \
apt-get install -y postgresql-server-dev-$PG_MAJOR \
git gcc make autoconf \
libc6-dev flex libcurl4-gnutls-dev \
libicu-dev libkrb5-dev liblz4-dev \
libpam0g-dev libreadline-dev libselinux1-dev\
libssl-dev libxslt1-dev libzstd-dev uuid-dev \
&& git clone -b "main" https://github.com/citusdata/citus.git \
&& MAKEFLAGS="-j $(grep -c ^processor /proc/cpuinfo)" \
&& cd citus && ./configure && make install && cd ../ && rm -rf /citus; \
else \
echo "deb [signed-by=/etc/apt/trusted.gpg.d/citusdata_community.gpg] https://packagecloud.io/citusdata/community/debian/ $(lsb_release -cs) main" > /etc/apt/sources.list.d/citusdata_community.list \
&& curl -sL https://packagecloud.io/citusdata/community/gpgkey | gpg --dearmor > /etc/apt/trusted.gpg.d/citusdata_community.gpg \
&& apt-get update -y \
&& apt-get -y install postgresql-$PG_MAJOR-citus-12.1; \
fi \
\
# Cleanup all locales but en_US.UTF-8
&& find /usr/share/i18n/charmaps/ -type f ! -name UTF-8.gz -delete \
&& find /usr/share/i18n/locales/ -type f ! -name en_US ! -name en_GB ! -name i18n* ! -name iso14651_t1 ! -name iso14651_t1_common ! -name 'translit_*' -delete \
&& echo 'en_US.UTF-8 UTF-8' > /usr/share/i18n/SUPPORTED \
\
# Make sure we have a en_US.UTF-8 locale available
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
\
# haproxy dummy config
&& echo 'global\n stats socket /run/haproxy/admin.sock mode 660 level admin' > /etc/haproxy/haproxy.cfg \
\
# vim config
&& echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local \
\
# Prepare postgres/patroni/haproxy environment
&& mkdir -p $PGHOME/.config/patroni /patroni /run/haproxy \
&& ln -s ../../postgres0.yml $PGHOME/.config/patroni/patronictl.yaml \
&& ln -s /patronictl.py /usr/local/bin/patronictl \
&& sed -i "s|/var/lib/postgresql.*|$PGHOME:/bin/bash|" /etc/passwd \
&& chown -R postgres:postgres /var/log \
\
# Download etcd
&& curl -sL https://github.com/coreos/etcd/releases/download/v${ETCDVERSION}/etcd-v${ETCDVERSION}-linux-$(dpkg --print-architecture).tar.gz \
| tar xz -C /usr/local/bin --strip=1 --wildcards --no-anchored etcd etcdctl \
\
&& if [ $(dpkg --print-architecture) = 'arm64' ]; then \
# Build confd
curl -sL https://go.dev/dl/go1.20.4.linux-arm64.tar.gz | tar xz -C /usr/local go \
&& export GOROOT=/usr/local/go && export PATH=$PATH:$GOROOT/bin \
&& git clone --recurse-submodules https://github.com/kelseyhightower/confd.git \
&& make -C confd \
&& cp confd/bin/confd /usr/local/bin/confd \
&& rm -rf /confd /usr/local/go; \
else \
# Download confd
curl -sL "https://github.com/kelseyhightower/confd/releases/download/v$CONFDVERSION/confd-$CONFDVERSION-linux-$(dpkg --print-architecture)" \
> /usr/local/bin/confd && chmod +x /usr/local/bin/confd; \
fi \
# Prepare client cert for HAProxy
&& cat /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/certs/ssl-cert-snakeoil.pem > /etc/ssl/private/ssl-cert-snakeoil.crt \
\
# Clean up all useless packages and some files
&& apt-get purge -y --allow-remove-essential python3-pip gzip bzip2 util-linux e2fsprogs \
libmagic1 bsdmainutils login ncurses-bin libmagic-mgc e2fslibs bsdutils \
exim4-config gnupg-agent dirmngr \
postgresql-server-dev-$PG_MAJOR git gcc make autoconf \
libc6-dev flex libicu-dev libkrb5-dev liblz4-dev \
libpam0g-dev libreadline-dev libselinux1-dev libssl-dev libxslt1-dev libzstd-dev uuid-dev \
&& apt-get autoremove -y \
&& apt-get clean -y \
&& rm -rf /var/lib/apt/lists/* \
/root/.cache \
/var/cache/debconf/* \
/etc/rc?.d \
/etc/systemd \
/docker-entrypoint* \
/sbin/pam* \
/sbin/swap* \
/sbin/unix* \
/usr/local/bin/gosu \
/usr/sbin/[acgipr]* \
/usr/sbin/*user* \
/usr/share/doc* \
/usr/share/man \
/usr/share/info \
/usr/share/i18n/locales/translit_hangul \
/usr/share/locale/?? \
/usr/share/locale/??_?? \
/usr/share/postgresql/*/man \
/usr/share/postgresql-common/pg_wrapper \
/usr/share/vim/vim*/doc \
/usr/share/vim/vim*/lang \
/usr/share/vim/vim*/tutor \
# /var/lib/dpkg/info/* \
&& find /usr/bin -xtype l -delete \
&& find /var/log -type f -exec truncate --size 0 {} \; \
&& find /usr/lib/python3/dist-packages -name '*test*' | xargs rm -fr \
&& find /lib/$(uname -m)-linux-gnu/security -type f ! -name pam_env.so ! -name pam_permit.so ! -name pam_unix.so -delete
# perform compression if it is necessary
ARG COMPRESS
RUN if [ "$COMPRESS" = "true" ]; then \
set -ex \
# Allow certain sudo commands from postgres
&& echo 'postgres ALL=(ALL) NOPASSWD: /bin/tar xpJf /a.tar.xz -C /, /bin/rm /a.tar.xz, /bin/ln -snf dash /bin/sh' >> /etc/sudoers \
&& ln -snf busybox /bin/sh \
&& arch=$(uname -m) \
&& darch=$(uname -m | sed 's/_/-/') \
&& files="/bin/sh /usr/bin/sudo /usr/lib/sudo/sudoers.so /lib/$arch-linux-gnu/security/pam_*.so" \
&& libs="$(ldd $files | awk '{print $3;}' | grep '^/' | sort -u) /lib/ld-linux-$darch.so.* /lib/$arch-linux-gnu/ld-linux-$darch.so.* /lib/$arch-linux-gnu/libnsl.so.* /lib/$arch-linux-gnu/libnss_compat.so.* /lib/$arch-linux-gnu/libnss_files.so.*" \
&& (echo /var/run $files $libs | tr ' ' '\n' && realpath $files $libs) | sort -u | sed 's/^\///' > /exclude \
&& find /etc/alternatives -xtype l -delete \
&& save_dirs="usr lib var bin sbin etc/ssl etc/init.d etc/alternatives etc/apt" \
&& XZ_OPT=-e9v tar -X /exclude -cpJf a.tar.xz $save_dirs \
# we call "cat /exclude" to avoid including files from the $save_dirs that are also among
# the exceptions listed in the /exclude, as "uniq -u" eliminates all non-unique lines.
# By calling "cat /exclude" a second time we guarantee that there will be at least two lines
# for each exception and therefore they will be excluded from the output passed to 'rm'.
&& /bin/busybox sh -c "(find $save_dirs -not -type d && cat /exclude /exclude && echo exclude) | sort | uniq -u | xargs /bin/busybox rm" \
&& /bin/busybox --install -s \
&& /bin/busybox sh -c "find $save_dirs -type d -depth -exec rmdir -p {} \; 2> /dev/null"; \
else \
/bin/busybox --install -s; \
fi
FROM scratch
COPY --from=builder / /
LABEL maintainer="Alexander Kukushkin <akukushkin@microsoft.com>"
ARG PG_MAJOR
ARG COMPRESS
ARG PGHOME
ARG PGDATA
ARG LC_ALL
ARG LANG
ARG PGBIN=/usr/lib/postgresql/$PG_MAJOR/bin
ENV LC_ALL=$LC_ALL LANG=$LANG EDITOR=/usr/bin/editor
ENV PGDATA=$PGDATA PATH=$PATH:$PGBIN
ENV ETCDCTL_API=3
COPY patroni /patroni/
COPY extras/confd/conf.d/haproxy.toml /etc/confd/conf.d/
COPY extras/confd/templates/haproxy-citus.tmpl /etc/confd/templates/haproxy.tmpl
COPY patroni*.py docker/entrypoint.sh /
COPY postgres?.yml $PGHOME/
WORKDIR $PGHOME
RUN sed -i 's/env python/&3/' /patroni*.py \
# "fix" patroni configs
&& sed -i 's/^ listen: 127.0.0.1/ listen: 0.0.0.0/' postgres?.yml \
&& sed -i "s|^\( data_dir: \).*|\1$PGDATA|" postgres?.yml \
&& sed -i "s|^#\( bin_dir: \).*|\1$PGBIN|" postgres?.yml \
&& sed -i 's/^ - encoding: UTF8/ - locale: en_US.UTF-8\n&/' postgres?.yml \
&& sed -i 's/^scope:/log:\n loggers:\n patroni.postgresql.citus: DEBUG\n#&/' postgres?.yml \
&& sed -i 's/^\(name\|etcd\| host\| authentication\| connect_address\| parameters\):/#&/' postgres?.yml \
&& sed -i 's/^ \(replication\|superuser\|rewind\|unix_socket_directories\|\(\( \)\{0,1\}\(username\|password\)\)\):/#&/' postgres?.yml \
&& sed -i 's/^postgresql:/&\n basebackup:\n checkpoint: fast/' postgres?.yml \
&& sed -i 's|^ parameters:|&\n max_connections: 100\n shared_buffers: 16MB\n ssl: "on"\n ssl_ca_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n ssl_cert_file: /etc/ssl/certs/ssl-cert-snakeoil.pem\n ssl_key_file: /etc/ssl/private/ssl-cert-snakeoil.key\n citus.node_conninfo: "sslrootcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslkey=/etc/ssl/private/ssl-cert-snakeoil.key sslcert=/etc/ssl/certs/ssl-cert-snakeoil.pem sslmode=verify-ca"|' postgres?.yml \
&& sed -i 's/^ pg_hba:/&\n - local all all trust/' postgres?.yml \
&& sed -i 's/^\(.*\) \(.*\) \(.*\) \(.*\) \(.*\) md5.*$/\1 hostssl \3 \4 all md5 clientcert=verify-ca/' postgres?.yml \
&& sed -i 's/^#\(ctl\| certfile\| keyfile\)/\1/' postgres?.yml \
&& sed -i 's|^# cafile: .*$| verify_client: required\n cafile: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \
&& sed -i 's|^# cacert: .*$| cacert: /etc/ssl/certs/ssl-cert-snakeoil.pem|' postgres?.yml \
&& sed -i 's/^# insecure: .*/ insecure: on/' postgres?.yml \
# client cert for HAProxy to access Patroni REST API
&& if [ "$COMPRESS" = "true" ]; then chmod u+s /usr/bin/sudo; fi \
&& chmod +s /bin/ping \
&& chown -R postgres:postgres $PGHOME /run /etc/haproxy
USER postgres
ENTRYPOINT ["/bin/sh", "/entrypoint.sh"]