Changes for end-to-end tests (#4884)

config/end_to_end/nsrlsvr.Dockerfile

@@ -0,0 +1,32 @@
+FROM ubuntu:jammy
+MAINTAINER Log2Timeline <log2timeline-dev@go
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Combining the apt-get commands into a single run reduces the size of the resulting image.
+# The apt-get installations below are interdependent and need to be done in sequence.
+RUN apt-get -y update && \
+    apt-get -y install apt-transport-https apt-utils && \
+    apt-get -y install libterm-readline-gnu-perl software-properties-common && \
+    apt-get -y install locales
+
+# Set terminal to UTF-8 by default.
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
+# Install nsrlsvr.
+RUN add-apt-repository ppa:gift/dev -y && \
+    apt-get update -q && \
+    apt-get install -y nsrlsvr-server
+
+# Initialize nsrlsvr hashes.txt file.
+RUN mkdir -p /var/share/nsrlsvr && \
+    mkdir -p /usr/share/nsrlsvr && \
+    touch /usr/share/nsrlsvr/hashes.txt
+
+WORKDIR /home/test/
+
+# Clean up apt-get cache files.
+RUN apt-get clean && rm -rf /var/cache/apt/* /var/lib/apt/lists/*

config/end_to_end/run_tests_with_docker.sh

@@ -73,25 +73,10 @@ do
 
 	# TODO: move custom test setup and teardown scripts to configuration parameter?
 
-	if [[ ${OUTPUT_FORMAT} == "opensearch" ]] || [[ ${OUTPUT_FORMAT} == "opensearch_ts" ]];
-	then
-		# Install OpenSearch and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_opensearch.sh && sleep 3m && ${COMMAND}";
-	fi
-
 	if [[ ${TEST_NAME} == "acserver-mounted" ]];
 	then
 		COMMAND="mkdir -p /mnt/acserver_mount && mount -o ro,noload,noacl,loop,offset=1048576 /sources/acserver.dd /mnt/acserver_mount && ./tests/end-to-end.py --config /config/${TEST_NAME}.ini --references-directory test_data/end_to_end --results-directory plaso-out --sources-directory /mnt --scripts-directory plaso/scripts && umount /mnt/acserver_mount && rmdir /mnt/acserver_mount";
 
-	elif [[ ${TEST_NAME} == *\-nsrlsvr ]];
-	then
-		# Install nsrlsvr and give it 3 minutes to start-up before running the output end-to-end test.
-		COMMAND="./config/linux/ubuntu_install_nsrlsvr.sh && sleep 3m && ${COMMAND}";
-
-	elif [[ ${TEST_NAME} == *\-redis ]];
-	then
-		# TODO: add support for Redis tests
-		continue;
 	fi
 	echo "Running ${TEST_CASE} end-to-end test: ${TEST_NAME}";
 

config/jenkins/build_nsrlsvr.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+#
+# Script to run nsrlsvr on an Ubuntu Jenkins instance with Docker.
+
+AUXILIARY_DATA_PATH="/media/auxiliary";
+
+# Exit on error.
+set -e
+
+sudo apt-get install -y curl unzip
+
+cd config/end_to_end;
+
+mkdir -p data;
+
+if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" ];
+then
+	# Note that NSRLFile.txt is approximate 4 GiB in size.
+	cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/NSRLFile.txt" data/
+fi
+
+if [ ! -f data/NSRLFile.txt ];
+then
+	if [ -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" ];
+	then
+		# Note that this is an older rds_modernm.zip that is approximate 2 GiB in size.
+		cp -f "${AUXILIARY_DATA_PATH}/nsrlsvr/rds_modernm.zip" data/
+	fi
+
+	if [ ! -f data/rds_modernm.zip ];
+	then
+		# Download the minimum modern RDS hash set.
+		# Note that rds_modernm.zip is approximate 18 GiB in size.
+		curl -o data/rds_modernm.zip https://s3.amazonaws.com/rds.nsrl.nist.gov/RDS/rds_2024.03.1/RDS_2024.03.1_modern_minimal.zip
+	fi
+
+	if [ ! -f data/rds_modernm.zip ];
+	then
+		echo "Missing: rds_modernm.zip";
+
+		exit 1
+	fi
+
+	unzip -x data/rds_modernm.zip data/rds_modernm/NSRLFile.txt
+
+	mv data/rds_modernm/NSRLFile.txt data/
+fi
+
+if [ ! -f data/NSRLFile.txt ];
+then
+	echo "Missing: NSRLFile.txt";
+
+	exit 1
+fi
+
+docker build -f nsrlsvr.Dockerfile --force-rm --no-cache -t log2timeline/nsrlsvr . ;
+
+# Update the nsrlsvr hashes.txt file from NSRLFile.txt
+docker run -v "${PWD}/data:/data:z" log2timeline/nsrlsvr /bin/bash -c "/usr/bin/python3 /usr/bin/nsrlupdate /data/NSRLFile.txt";
+
+# Preserver the intermediate container so we don't have to rebuild hashes.txt
+docker commit `docker ps -lq` | cut -c8- > nsrlsvr.container
+

config/jenkins/greendale/psort-studentpc1-nsrlsvr.ini

@@ -1,6 +1,6 @@
 [nsrlsvr]
 case=analyze_and_output
-analysis_options=--analysis nsrlsvr --nsrlsvr-port=9120
+analysis_options=--analysis nsrlsvr --nsrlsvr-host=nsrlsvr --nsrlsvr-port=9120
 source=studentpc1.plaso
 output_file=studentpc1.csv
 output_format=dynamic

config/jenkins/run_end_to_end_tests.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to run end-to-end tests on a Linux Jenkins instance with Docker.
+# Script to run end-to-end tests on an Ubuntu Jenkins instance with Docker.
 
 # Fail on error.
 set -e
@@ -33,7 +33,11 @@ docker run log2timeline/plaso ./utils/check_dependencies.py;
 
 COMMAND="./tests/end-to-end.py --config /config/${CONFIGURATION_NAME}.ini --references-directory test_data/end_to_end --results-directory /home/test/plaso/plaso-out --sources-directory /sources --scripts-directory plaso/scripts";
 
-if test ${CONFIGURATION_NAME} = "output_opensearch";
+if test ${CONFIGURATION_NAME} = "psort-studentpc1-nsrlsvr";
+then
+	DOCKER_NETWORK="--network=nsrlsvr-network";
+
+elif test ${CONFIGURATION_NAME} = "output_opensearch" || test ${CONFIGURATION_NAME} = "output_opensearch_ts";
 then
 	DOCKER_NETWORK="--network=opensearch-network";