Use new events by default (#2007)

* Use new events for testsuite ci tests

* Fix circleci tests

* Update docs

build_goreleaser/eventingester/Dockerfile

@@ -1,12 +1,8 @@
 ARG BASE_IMAGE=alpine:3.17.0
 FROM ${BASE_IMAGE}
-
 RUN addgroup -S -g 2000 armada && adduser -S -u 1000 armada -G armada
 USER armada
-
 COPY eventingester /app/
 COPY config/eventingester/config.yaml /app/config/eventingester/config.yaml
-
 WORKDIR /app
-
 ENTRYPOINT ["./eventingester"]

config/armada/config.yaml

@@ -32,7 +32,7 @@ eventsApiRedis:
   password: ""
   db: 1
   poolSize: 1000
-defaultToLegacyEvents: true
+forceNewEvents: true
 scheduling:
   preemption:
     enabled: true

config/eventingester/config.yaml

@@ -2,7 +2,7 @@ redis:
   addrs:
     - redis:6379
   password: ""
-  db: 0
+  db: 1
   poolSize: 1000
 pulsar:
   URL: pulsar://pulsar:6650
@@ -12,7 +12,7 @@ pulsar:
 subscriptionName: "events-ingester"
 minMessageCompressionSize: 1024
 batchSize: 1048576  #1MB
-batchDuration: 500ms
+batchDuration: 100ms
 batchMessages: 10000
 eventRetentionPolicy:
   expiryEnabled: true

docker-compose.yaml

@@ -67,6 +67,7 @@ services:
       - postgres
       - pulsar
       - stan
+      - eventingester
     working_dir: /app
     entrypoint: ./server
     command: --config /config/insecure-armada-auth-config.yaml
@@ -96,8 +97,6 @@ services:
       - kind
     ports:
       - 8082:8082
-    depends_on:
-      - server
     volumes:
       - ./.kube/internal:/.kube
     environment:
@@ -124,6 +123,7 @@ services:
       - kind
     depends_on:
       - postgres
+      - pulsar
     volumes:
       - ./localdev/config:/config
     working_dir: /app
@@ -137,5 +137,6 @@ services:
       - kind
     depends_on:
       - redis
+      - pulsar
     working_dir: /app
-    entrypoint: eventingester
+    entrypoint: ./eventingester

docs/development_guide.md

@@ -24,7 +24,7 @@ mage Kind
 # Start necessary dependencies.
 # On Arm-based Macs, you may need to change the pulsar image
 # in docker-compose.yaml to be kezhenxu94/pulsar.
-docker-compose up -d redis stan postgres pulsar
+docker-compose up -d redis stan postgres pulsar eventingester
 
 # Verify that dependencies started successfully
 # (check that redis, stan, postgres, and pulsar are all up).

makefile

@@ -393,7 +393,7 @@ rebuild-executor: build-docker-executor
 
 .ONESHELL:
 tests-e2e-teardown:
-	docker rm -f nats redis pulsar server executor postgres lookout-ingester-migrate lookout-ingester jobservice || true
+	docker rm -f nats redis pulsar server executor postgres lookout-ingester-migrate lookout-ingester jobservice event-ingester || true
 	kind delete cluster --name armada-test || true
 	rm .kube/config || true
 	rmdir .kube || true
@@ -451,8 +451,9 @@ tests-e2e-setup: setup-cluster
 		armada-executor --config /e2e/setup/insecure-executor-config.yaml
 	docker run -d --name lookout-ingester-migrate  --network=kind -v ${PWD}/e2e:/e2e \
 		armada-lookout-ingester --config /e2e/setup/lookout-ingester-config.yaml --migrateDatabase
-	docker run -d --name lookout-ingester  --network=kind -v ${PWD}/e2e:/e2e \
+	docker run -d --name lookout-ingester --network=kind -v ${PWD}/e2e:/e2e \
 		armada-lookout-ingester --config /e2e/setup/lookout-ingester-config.yaml
+	docker run -d --name event-ingester --network=kind -v ${PWD}/e2e:/e2e armada-event-ingester
 	docker run -d --name jobservice --network=kind -v ${PWD}/e2e:/e2e \
 	    armada-jobservice run --config /e2e/setup/jobservice.yaml