video: add recording of video+ prefixes (#737)

* [AUTO-COMMIT] Update `manifest.yaml`

Files changed:
M	manifest.yaml

* video: add recording of video+ prefixes

* Makefile: remove old docker-local dependency

* Dockerfile: download catalyst with curl

* ci: try making box

* ci: no frontend in tests

* go.mod: use upgraded, forked testcontainers-go

We're currently hitting this issue testcontainers/testcontainers-go#1783
and the forked version fixes it. When that fix is upstreamed we may move
back to an official build.

* ci: disable ghcr.io repos

they're only there as a backup and they result in this alllll the time

ERROR: failed to solve: failed to compute cache key: failed to copy: read tcp 172.17.0.2:44614->20.60.63.1:443: read: connection timed out
Error: buildx failed with: ERROR: failed to solve: failed to compute cache key: failed to copy: read tcp 172.17.0.2:44614->20.60.63.1:443: read: connection timed out

* Dockerfile: bypass cloudflare for catalyst pull

---------

Co-authored-by: livepeer-docker <livepeer-docker@users.noreply.github.com>

.github/workflows/catalyst.yaml

@@ -57,13 +57,13 @@ jobs:
             refs: |
               livepeer/catalyst
               livepeerci/catalyst
-              ghcr.io/livepeer/catalyst
+            # ghcr.io/livepeer/catalyst
           - target: livepeer-in-a-box
             label: livepeer/in-a-box
             refs: |
               livepeer/in-a-box
               livepeerci/in-a-box
-              ghcr.io/livepeer/in-a-box
+            # ghcr.io/livepeer/in-a-box
 
     steps:
       - name: Set up QEMU

.github/workflows/test.yaml

@@ -60,6 +60,9 @@ jobs:
       - name: Build Docker image
         run: make docker DOCKER_TAG=catalyst-e2e-test
 
+      - name: Build Docker Box image
+        run: make box
+
       - name: Run E2E tests
         run:
           go test $(go list ./... | grep 'test/e2e') --timeout 15m --image

Dockerfile

@@ -8,7 +8,10 @@ ARG TARGETARCH
 
 # Download c2patool needed to sign our C2PA manifest
 # We download it from any of our previous builds, because building c2patool from source is very slow with QEMU
-ADD https://build.livepeer.live/catalyst/772a57003e6d96c9a47ef18fccb51a0c61207074/livepeer-catalyst-linux-${TARGETARCH}.tar.gz /catalyst.tar.gz
+RUN	apt update && apt install -yqq \
+	curl \
+	ca-certificates
+RUN curl https://storage.googleapis.com/build.livepeer.live/catalyst/772a57003e6d96c9a47ef18fccb51a0c61207074/livepeer-catalyst-linux-${TARGETARCH}.tar.gz -o /catalyst.tar.gz
 RUN tar xzf /catalyst.tar.gz
 
 WORKDIR	/src

Makefile

@@ -174,7 +174,7 @@ docker:
 		.
 
 .PHONY: docker-local
-docker-local: downloader scripts 
+docker-local: scripts
 	tar ch ./bin ./config \
 	| docker buildx build \
 		--load \

config/full-stack.json

@@ -8,6 +8,10 @@
     [
       "videorec+",
       "s3+http://admin:password@localhost:9000/os-recordings/$wildcard/$uuid/source/$segmentCounter.ts?m3u8=../output.m3u8&split=5&video=source&audio=AAC&append=1&waittrackcount=2&recstart=-1"
+    ],
+    [
+      "video+",
+      "s3+http://admin:password@localhost:9000/os-recordings/$wildcard/$uuid/source/$segmentCounter.ts?m3u8=../output.m3u8&split=5&video=source&audio=AAC&append=1&waittrackcount=2&recstart=-1"
     ]
   ],
   "config": {

go.mod

@@ -2,23 +2,25 @@ module github.com/livepeer/catalyst
 
 go 1.19
 
+replace github.com/testcontainers/testcontainers-go v0.26.0 => github.com/lefinal/testcontainers-go v0.0.0-20231107224233-ca049655293f
+
 require (
 	github.com/ProtonMail/gopenpgp/v2 v2.4.10
-	github.com/golang/glog v1.1.1
+	github.com/golang/glog v1.1.2
 	github.com/livepeer/stream-tester v0.12.30-0.20230823234013-5cfb4bbcf27d
 	github.com/magicsong/color-glog v0.0.1
 	github.com/minio/minio-go/v7 v7.0.46
 	github.com/peterbourgon/ff/v3 v3.3.1
 	github.com/stretchr/testify v1.8.4
-	github.com/testcontainers/testcontainers-go v0.20.1-0.20230510111141-38b35a314fe4
+	github.com/testcontainers/testcontainers-go v0.26.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/compute v1.20.0 // indirect
+	cloud.google.com/go v0.110.10 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.0 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
 	cloud.google.com/go/storage v1.30.1 // indirect
 	contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
@@ -34,11 +36,11 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cloudflare/circl v1.1.0 // indirect
-	github.com/containerd/containerd v1.7.1 // indirect
+	github.com/containerd/containerd v1.7.9 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/docker v24.0.2+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
@@ -51,12 +53,11 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/btree v1.0.0 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
-	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/uuid v1.4.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/gosuri/uilive v0.0.3 // indirect
 	github.com/gosuri/uiprogress v0.0.1 // indirect
@@ -69,10 +70,9 @@ require (
 	github.com/hashicorp/logutils v1.0.0 // indirect
 	github.com/hashicorp/memberlist v0.5.0 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
-	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.16.5 // indirect
+	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/livepeer/catalyst-api v0.1.1 // indirect
@@ -87,15 +87,15 @@ require (
 	github.com/miekg/dns v1.1.50 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/sha256-simd v1.0.1 // indirect
-	github.com/moby/patternmatcher v0.5.0 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
-	github.com/opencontainers/runc v1.1.7 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc5 // indirect
+	github.com/opencontainers/runc v1.1.10 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/peterbourgon/ff/v2 v2.0.1 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
@@ -111,31 +111,43 @@ require (
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/net v0.11.0 // indirect
-	golang.org/x/oauth2 v0.9.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/tools v0.9.1 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/oauth2 v0.13.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/tools v0.15.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.129.0 // indirect
+	google.golang.org/api v0.149.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/grpc v1.56.1 // indirect
+	google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gotest.tools/v3 v3.5.0 // indirect
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20220822140716-1678d6eb0cbe // indirect
 	github.com/ProtonMail/go-mime v0.0.0-20220302105931-303f85f7fe0f // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/livepeer/livepeer-data v0.7.3 // indirect
+	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/sirupsen/logrus v1.9.2 // indirect
-	golang.org/x/crypto v0.10.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
-	golang.org/x/text v0.10.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
+	github.com/shirou/gopsutil/v3 v3.23.10 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 )