Skip to content

Commit

Permalink
Add DL agent
Browse files Browse the repository at this point in the history
  • Loading branch information
angelini committed May 8, 2024
1 parent cc8f6bb commit a0ea5f2
Show file tree
Hide file tree
Showing 11 changed files with 439 additions and 10 deletions.
4 changes: 3 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM --platform=linux/amd64 registry.fedoraproject.org/fedora-minimal:40

RUN microdnf install -y curl findutils gzip iputils less postgresql procps shadow-utils tar time which \
RUN microdnf install -y curl findutils gzip hostname iputils less postgresql procps shadow-utils tar time which \
&& microdnf clean all

RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \
Expand All @@ -19,7 +19,9 @@ WORKDIR /home/main
RUN mkdir -p /home/main/secrets
VOLUME /home/main/secrets/tls
VOLUME /home/main/secrets/paseto
VOLUME /home/main/varlib

COPY release/client_linux_amd64 client
COPY release/server_linux_amd64 server
COPY migrations migrations
COPY entrypoint.sh entrypoint.sh
Expand Down
46 changes: 39 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,18 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto')
MIGRATE_DIR := ./migrations
SERVICE := $(PROJECT).server

K8S_NS := dateilager
K8S_CTX := docker-desktop
KC := kubectl --context $(K8S_CTX) -n $(K8S_NS)

.PHONY: install migrate migrate-create clean build lint release
.PHONY: test test-one test-fuzz test-js lint-js build-js
.PHONY: reset-db setup-local server server-profile install-js
.PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache
.PHONY: client-getcache client-gc-contents client-gc-project client-gc-random-projects
.PHONY: health upload-container-image run-container gen-docs
.PHONY: start-agent gen-docs health
.PHONY: upload-container-image upload-prerelease-container-image build-local-container run-container
.PHONY: setup-k8s reset-k8s deploy-k8s
.PHONY: load-test-new load-test-get load-test-update

install:
Expand Down Expand Up @@ -189,6 +195,14 @@ client-gc-random-projects: export DL_SKIP_SSL_VERIFICATION=1
client-gc-random-projects:
go run cmd/client/main.go gc --host $(GRPC_HOST) --mode random-projects --sample 25 --keep 1

start-agent: export DL_TOKEN=$(DEV_TOKEN_ADMIN)
start-agent: export DL_SKIP_SSL_VERIFICATION=1
start-agent:
go run cmd/client/main.go agent --host $(GRPC_HOST) --dir /tmp/dl_agent

gen-docs:
go run cmd/gen-docs/main.go

health:
grpc-health-probe -addr $(GRPC_SERVER)
grpc-health-probe -addr $(GRPC_SERVER) -service $(SERVICE)
Expand All @@ -206,12 +220,30 @@ upload-prerelease-container-image: release
docker build -t gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) .
docker push gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT)

run-container: release
docker build -t dl-local:latest .
docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl

gen-docs:
go run cmd/gen-docs/main.go
build-local-container: release
docker build -t local/dateilager:latest .

run-container: build-local-container
docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto local/dateilager:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl

setup-k8s:
kubectl --context $(K8S_CTX) apply -f k8s/namespace.yaml
$(KC) create secret tls dl-tls-secret --cert=development/server.crt --key=development/server.key
$(KC) create secret generic dl-paseto-secret --from-file=paseto.pub=development/paseto.pub
$(KC) create secret generic dl-app-secrets --from-literal="DATABASE_URL=postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432/dl"
$(KC) create secret generic dl-agent-secrets --from-literal="DL_TOKEN=$(DEV_TOKEN_ADMIN)"

reset-k8s:
$(KC) delete service --ignore-not-found dl-agent
$(KC) delete daemonset --ignore-not-found dl-agent
$(KC) delete service --ignore-not-found dl-headless
$(KC) delete deployment --ignore-not-found dl-server
$(KC) delete deployment --ignore-not-found --force dl-sandbox

deploy-k8s: build-local-container reset-k8s
$(KC) apply -f k8s/server.yaml
$(KC) apply -f k8s/agent.yaml
$(KC) apply -f k8s/sandbox.yaml

define load-test
ghz --cert=development/server.crt --key=development/server.key \
Expand Down
4 changes: 2 additions & 2 deletions internal/files/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func writeObject(rootDir string, cacheObjectsDir string, reader *db.TarReader, h
return err
}
hashHex := hex.EncodeToString(content)
return hardlinkDir(filepath.Join(cacheObjectsDir, hashHex, header.Name), path)
return HardlinkDir(filepath.Join(cacheObjectsDir, hashHex, header.Name), path)

case tar.TypeReg:
dir := filepath.Dir(path)
Expand Down Expand Up @@ -193,7 +193,7 @@ func makeSymlink(oldname, newname string) error {
return nil
}

func hardlinkDir(olddir, newdir string) error {
func HardlinkDir(olddir, newdir string) error {
if fileExists(newdir) {
err := os.RemoveAll(newdir)
if err != nil {
Expand Down
77 changes: 77 additions & 0 deletions k8s/agent.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: dl-agent
labels:
app: dl-agent
spec:
selector:
matchLabels:
app: dl-agent
template:
metadata:
name: dl-agent
labels:
app: dl-agent
spec:
containers:
- name: agent
image: local/dateilager:latest
imagePullPolicy: Never
securityContext: # FIXME
allowPrivilegeEscalation: false
runAsUser: 0
command: ["./client"]
args:
[
"agent",
"--host=dl-headless.dateilager.svc.cluster.local",
"--headless-host=dl-headless.dateilager.svc.cluster.local",
"--dir=/home/main/varlib/dl_cache",
"--log-level=info",
"--log-encoding=json",
]
ports:
- name: http
containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /healthz
port: 8080
initialDelaySeconds: 3
livenessProbe:
httpGet:
path: /healthz
port: 8080
initialDelaySeconds: 5
periodSeconds: 2
failureThreshold: 2
env:
- name: DL_SKIP_SSL_VERIFICATION
value: "1"
envFrom:
- secretRef:
name: dl-agent-secrets
volumeMounts:
- name: varlib
mountPath: /home/main/varlib
volumes:
- name: varlib
hostPath:
path: /var/lib
---
apiVersion: v1
kind: Service
metadata:
name: dl-agent
labels:
app: dl-agent
spec:
internalTrafficPolicy: Local
selector:
app: dl-agent
ports:
- name: http
port: 8080
targetPort: 8080
4 changes: 4 additions & 0 deletions k8s/namespace.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: dateilager
40 changes: 40 additions & 0 deletions k8s/sandbox.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dl-sandbox
labels:
app: dl-sandbox
spec:
replicas: 1
selector:
matchLabels:
app: dl-sandbox
template:
metadata:
name: dl-sandbox
labels:
app: dl-sandbox
spec:
containers:
- name: sandbox
image: local/dateilager:latest
imagePullPolicy: Never
command: ["bash", "-c", "--"]
args: ["while true; do sleep 30; done;"]
# command: ["bash"]
# args:
# [
# "-c",
# 'curl -XPOST -H ''Content-Type: application/json'' -d "{\"uid\":\"${K8S_CONTAINER_ID}\", \"volume\": \"appdir\"}" dl-agent.dateilager.svc.cluster.local:8080/link_cache',
# ]
env:
- name: K8S_CONTAINER_ID
valueFrom:
fieldRef:
fieldPath: metadata.uid
volumeMounts:
- name: appdir
mountPath: /tmp/appdir
volumes:
- name: appdir
emptyDir: {}
94 changes: 94 additions & 0 deletions k8s/server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dl-server
labels:
app: dl-server
spec:
replicas: 1
selector:
matchLabels:
app: dl-server
template:
metadata:
name: dl-server
labels:
app: dl-server
spec:
containers:
- name: server
image: local/dateilager:latest
imagePullPolicy: Never
command: ["./server"]
args:
[
"--log-level=info",
"--log-encoding=json",
"--port=5051",
"--dburi=$(DATABASE_URL)",
"--cert=secrets/tls/tls.crt",
"--key=secrets/tls/tls.key",
"--paseto=secrets/paseto/paseto.pub",
]
ports:
- name: api
containerPort: 5051
protocol: TCP
readinessProbe:
exec:
command:
[
"/bin/grpc_health_probe",
"-addr=:5051",
"-service=dateilager.server",
"-tls",
"-tls-no-verify",
]
initialDelaySeconds: 1
livenessProbe:
exec:
command:
[
"/bin/grpc_health_probe",
"-addr=:5051",
"-service=dateilager.server",
"-tls",
"-tls-no-verify",
]
initialDelaySeconds: 2
periodSeconds: 2
failureThreshold: 2
env:
- name: DL_ENV
value: "dev"
envFrom:
- secretRef:
name: dl-app-secrets
volumeMounts:
- mountPath: "/home/main/secrets/tls"
name: tls-secret
- mountPath: "/home/main/secrets/paseto"
name: paseto-secret
volumes:
- name: tls-secret
secret:
secretName: dl-tls-secret
- name: paseto-secret
secret:
secretName: dl-paseto-secret
---
apiVersion: v1
kind: Service
metadata:
labels:
app: dl-headless
name: dl-headless
spec:
clusterIP: None
selector:
app: dl-server
ports:
- name: grpc
protocol: TCP
port: 5051
targetPort: 5051

0 comments on commit a0ea5f2

Please sign in to comment.