Merge pull request #134 from CuriBio/mc-comm-parse-data

Mc comm parse data

Pipfile

@@ -37,6 +37,7 @@ mantarray-waveform-analysis = "==0.6.0"
 scipy = "==1.6.1"
 secrets-manager = "==0.4"
 semver = "==2.13.0"
+pyserial = "==3.5"
 stdlib-utils = "==0.4.3"
 xem-wrapper = "==0.2.5"
 

src/mantarray_desktop_app/__init__.py

@@ -6,6 +6,7 @@
 from . import file_writer
 from . import firmware_manager
 from . import main
+from . import mc_comm
 from . import mc_simulator
 from . import ok_comm
 from . import process_manager
@@ -74,6 +75,8 @@
 from .constants import REFERENCE_VOLTAGE
 from .constants import ROUND_ROBIN_PERIOD
 from .constants import SECONDS_TO_WAIT_WHEN_POLLING_QUEUES
+from .constants import SERIAL_COMM_ADDITIONAL_BYTES_INDEX
+from .constants import SERIAL_COMM_BAUD_RATE
 from .constants import SERIAL_COMM_CHECKSUM_FAILURE_PACKET_TYPE
 from .constants import SERIAL_COMM_CHECKSUM_LENGTH_BYTES
 from .constants import SERIAL_COMM_COMMAND_RESPONSE_PACKET_TYPE
@@ -119,6 +122,9 @@
 from .exceptions import MultiprocessingNotSetToSpawnError
 from .exceptions import RecordingFolderDoesNotExistError
 from .exceptions import ScriptDoesNotContainEndCommandError
+from .exceptions import SerialCommIncorrectChecksumFromInstrumentError
+from .exceptions import SerialCommIncorrectChecksumFromPCError
+from .exceptions import SerialCommIncorrectMagicWordFromMantarrayError
 from .exceptions import SerialCommPacketRegistrationReadEmptyError
 from .exceptions import SerialCommPacketRegistrationSearchExhaustedError
 from .exceptions import SerialCommPacketRegistrationTimoutError
@@ -154,7 +160,6 @@
 from .mantarray_front_panel import MantarrayFrontPanel
 from .mantarray_front_panel import MantarrayFrontPanelMixIn
 from .mc_comm import McCommunicationProcess
-from .mc_simulator import create_data_packet
 from .mc_simulator import MantarrayMcSimulator
 from .ok_comm import build_file_writer_objects
 from .ok_comm import check_barcode_for_errors
@@ -168,6 +173,8 @@
 from .process_manager import MantarrayProcessesManager
 from .process_monitor import MantarrayProcessesMonitor
 from .queue_container import MantarrayQueueContainer
+from .serial_comm_utils import create_data_packet
+from .serial_comm_utils import validate_checksum
 from .server import clear_the_server_thread
 from .server import flask_app
 from .server import get_api_endpoint
@@ -361,10 +368,17 @@
     "SERIAL_COMM_CHECKSUM_LENGTH_BYTES",
     "SERIAL_COMM_TIMESTAMP_LENGTH_BYTES",
     "SerialCommPacketRegistrationTimoutError",
+    "SerialCommIncorrectMagicWordFromMantarrayError",
     "SerialCommPacketRegistrationReadEmptyError",
     "SERIAL_COMM_MAX_PACKET_LENGTH_BYTES",
     "SerialCommPacketRegistrationSearchExhaustedError",
     "SERIAL_COMM_SIMPLE_COMMAND_PACKET_TYPE",
     "SERIAL_COMM_REBOOT_COMMAND_BYTE",
     "MC_REBOOT_DURATION_SECONDS",
+    "mc_comm",
+    "validate_checksum",
+    "SerialCommIncorrectChecksumFromInstrumentError",
+    "SERIAL_COMM_BAUD_RATE",
+    "SerialCommIncorrectChecksumFromPCError",
+    "SERIAL_COMM_ADDITIONAL_BYTES_INDEX",
 ]

src/mantarray_desktop_app/constants.py

@@ -231,6 +231,8 @@
 SECONDS_TO_WAIT_WHEN_POLLING_QUEUES = 0.02  # Due to the unreliablity of the .empty() .qsize() methods in queues, switched to a .get(timeout=) approach for polling the queues in the subprocesses.  Eli (10/26/20): 0.01 seconds was still causing sporadic failures in Linux CI in Github, so bumped to 0.02 seconds.
 
 # Serial Communication Values
+SERIAL_COMM_BAUD_RATE = 4000000
+
 MC_REBOOT_DURATION_SECONDS = 5
 
 SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS = 5
@@ -244,6 +246,7 @@
 
 SERIAL_COMM_MODULE_ID_INDEX = 18
 SERIAL_COMM_PACKET_TYPE_INDEX = 19
+SERIAL_COMM_ADDITIONAL_BYTES_INDEX = 20
 
 SERIAL_COMM_MAIN_MODULE_ID = 0
 SERIAL_COMM_STATUS_BEACON_PACKET_TYPE = 0

src/mantarray_desktop_app/exceptions.py

@@ -140,3 +140,15 @@ class SerialCommPacketRegistrationReadEmptyError(Exception):
 
 class SerialCommPacketRegistrationSearchExhaustedError(Exception):
     pass
+
+
+class SerialCommIncorrectChecksumFromInstrumentError(Exception):
+    pass
+
+
+class SerialCommIncorrectChecksumFromPCError(Exception):
+    pass
+
+
+class SerialCommIncorrectMagicWordFromMantarrayError(Exception):
+    pass

src/mantarray_desktop_app/mc_comm.py

@@ -2,17 +2,67 @@
 """Process controlling communication with Mantarray Microcontroller."""
 from __future__ import annotations
 
+import datetime
+from multiprocessing import Queue
+from time import perf_counter
 from time import sleep
 from typing import Any
 from typing import List
+from zlib import crc32
 
+import serial
+import serial.tools.list_ports as list_ports
+
+from .constants import SERIAL_COMM_ADDITIONAL_BYTES_INDEX
+from .constants import SERIAL_COMM_BAUD_RATE
+from .constants import SERIAL_COMM_CHECKSUM_FAILURE_PACKET_TYPE
+from .constants import SERIAL_COMM_CHECKSUM_LENGTH_BYTES
 from .constants import SERIAL_COMM_MAGIC_WORD_BYTES
+from .constants import SERIAL_COMM_MAIN_MODULE_ID
 from .constants import SERIAL_COMM_MAX_PACKET_LENGTH_BYTES
+from .constants import SERIAL_COMM_MODULE_ID_INDEX
+from .constants import SERIAL_COMM_PACKET_TYPE_INDEX
+from .constants import SERIAL_COMM_STATUS_BEACON_PACKET_TYPE
 from .constants import SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS
+from .exceptions import SerialCommIncorrectChecksumFromInstrumentError
+from .exceptions import SerialCommIncorrectChecksumFromPCError
+from .exceptions import SerialCommIncorrectMagicWordFromMantarrayError
 from .exceptions import SerialCommPacketRegistrationReadEmptyError
 from .exceptions import SerialCommPacketRegistrationSearchExhaustedError
 from .exceptions import SerialCommPacketRegistrationTimoutError
+from .exceptions import UnrecognizedSerialCommModuleIdError
+from .exceptions import UnrecognizedSerialCommPacketTypeError
 from .instrument_comm import InstrumentCommProcess
+from .mc_simulator import MantarrayMcSimulator
+from .serial_comm_utils import validate_checksum
+
+
+def _get_formatted_utc_now() -> str:
+    return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")
+
+
+def _get_seconds_since_read_start(start: float) -> float:
+    return perf_counter() - start
+
+
+def _process_main_module_comm(comm_from_instrument: bytes) -> None:
+    packet_type = comm_from_instrument[SERIAL_COMM_PACKET_TYPE_INDEX]
+    if packet_type == SERIAL_COMM_CHECKSUM_FAILURE_PACKET_TYPE:
+        returned_packet = (
+            SERIAL_COMM_MAGIC_WORD_BYTES
+            + comm_from_instrument[
+                SERIAL_COMM_ADDITIONAL_BYTES_INDEX:-SERIAL_COMM_CHECKSUM_LENGTH_BYTES
+            ]
+        )
+        raise SerialCommIncorrectChecksumFromPCError(returned_packet)
+
+    if packet_type == SERIAL_COMM_STATUS_BEACON_PACKET_TYPE:
+        pass
+    else:
+        module_id = comm_from_instrument[SERIAL_COMM_MODULE_ID_INDEX]
+        raise UnrecognizedSerialCommPacketTypeError(
+            f"Packet Type ID: {packet_type} is not defined for Module ID: {module_id}"
+        )
 
 
 class McCommunicationProcess(InstrumentCommProcess):
@@ -30,21 +80,117 @@ def __init__(self, *args: Any, **kwargs: Any):
             self._board_queues
         )
 
+    def _setup_before_loop(self) -> None:
+        msg = {
+            "communication_type": "log",
+            "message": f'Microcontroller Communication Process initiated at {datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")}',
+        }
+        to_main_queue = self._board_queues[0][1]
+        if not self._suppress_setup_communication_to_main:
+            to_main_queue.put(msg)
+        self.create_connections_to_all_available_boards()
+
+        board_idx = 0
+        board = self._board_connections[board_idx]
+        if isinstance(
+            board, MantarrayMcSimulator
+        ):  # pragma: no cover  # Tanner (3/16/21): it's impossible to connect to any serial port in CI, so _setup_before_loop will always be called with a simulator and this if statement will always be true in pytest
+            # Tanner (3/16/21): Current assumption is that a live mantarray will be running by the time we connect to it, so starting simulator here and waiting for it to complete start upt
+            board.start()
+            while not board.is_start_up_complete():
+                pass
+
     def is_registered_with_serial_comm(self, board_idx: int) -> bool:
         """Mainly for use in testing."""
         is_registered: bool = self._is_registered_with_serial_comm[board_idx]
         return is_registered
 
     def create_connections_to_all_available_boards(self) -> None:
-        raise NotImplementedError()  # Tanner (12/18/21): adding this as a placeholder for now to override abstract method. This method will be defined and the NotImplementedError removed before this class is instantied in any source code
+        """Create initial connections to boards.
+
+        If a board is not present, a simulator will be put in.
+        """
+        num_boards_connected = self.determine_how_many_boards_are_connected()
+        to_main_queue = self._board_queues[0][1]
+        for i in range(num_boards_connected):
+            msg = {
+                "communication_type": "board_connection_status_change",
+                "board_index": i,
+            }
+
+            for name in list(list_ports.comports()):
+                name = str(name)
+                # Tanner (3/15/21): As long as the STM eval board is used in the Mantarray, it will show up as so and we can look for the Mantarray by checking for STM in the name
+                if "STM" not in name:
+                    continue
+                port = name[-5:-1]  # parse out the name of the COM port
+                serial_obj = serial.Serial(
+                    port=port,
+                    baudrate=SERIAL_COMM_BAUD_RATE,
+                    bytesize=8,
+                    timeout=0,
+                    stopbits=serial.STOPBITS_ONE,
+                )
+                break
+            else:
+                msg["message"] = "No board detected. Creating simulator."
+                serial_obj = MantarrayMcSimulator(
+                    Queue(),
+                    Queue(),
+                    Queue(),
+                    Queue(),
+                )
+            self.set_board_connection(i, serial_obj)
+            # TODO Tanner (3/15/21): add serial number and nickname to msg
+            msg["is_connected"] = not isinstance(serial_obj, MantarrayMcSimulator)
+            msg["timestamp"] = _get_formatted_utc_now()
+            to_main_queue.put(msg)
 
     def _commands_for_each_run_iteration(self) -> None:
+        self._handle_incoming_data()
+
+    def _handle_incoming_data(self) -> None:
         board_idx = 0
-        if (
-            not self._is_registered_with_serial_comm[board_idx]
-            and self._board_connections[board_idx] is not None
-        ):
+        board = self._board_connections[board_idx]
+        if board is None:
+            return
+        if not self._is_registered_with_serial_comm[board_idx]:
             self._register_magic_word(board_idx)
+        elif board.in_waiting > 0:
+            magic_word_bytes = board.read(size=len(SERIAL_COMM_MAGIC_WORD_BYTES))
+            if magic_word_bytes != SERIAL_COMM_MAGIC_WORD_BYTES:
+                raise SerialCommIncorrectMagicWordFromMantarrayError(
+                    str(magic_word_bytes)
+                )
+        else:
+            return
+        packet_size_bytes = board.read(size=2)
+        packet_size = int.from_bytes(packet_size_bytes, byteorder="little")
+        data_packet_bytes = board.read(size=packet_size)
+        # TODO Tanner (3/15/21): eventually make sure the expected number of bytes are read. Need to figure out what to do if not enough bytes are read first
+
+        # validate checksum before handling the communication. Need to reconstruct the whole packet to get the correct checksum
+        full_data_packet = (
+            SERIAL_COMM_MAGIC_WORD_BYTES + packet_size_bytes + data_packet_bytes
+        )
+        is_checksum_valid = validate_checksum(full_data_packet)
+        if not is_checksum_valid:
+            calculated_checksum = crc32(
+                full_data_packet[:-SERIAL_COMM_CHECKSUM_LENGTH_BYTES]
+            )
+            received_checksum = int.from_bytes(
+                full_data_packet[-SERIAL_COMM_CHECKSUM_LENGTH_BYTES:],
+                byteorder="little",
+            )
+            raise SerialCommIncorrectChecksumFromInstrumentError(
+                f"Checksum Received: {received_checksum}, Checksum Calculated: {calculated_checksum}, Full Data Packet: {str(full_data_packet)}"
+            )
+
+        module_id = full_data_packet[SERIAL_COMM_MODULE_ID_INDEX]
+        if module_id == SERIAL_COMM_MAIN_MODULE_ID:
+            _process_main_module_comm(full_data_packet)
+        else:
+            raise UnrecognizedSerialCommModuleIdError(module_id)
 
     def _register_magic_word(self, board_idx: int) -> None:
         board = self._board_connections[board_idx]
@@ -54,9 +200,10 @@ def _register_magic_word(self, board_idx: int) -> None:
         magic_word_len = len(SERIAL_COMM_MAGIC_WORD_BYTES)
         magic_word_test_bytes = board.read(size=magic_word_len)
         magic_word_test_bytes_len = len(magic_word_test_bytes)
+        # wait for at least 8 bytes to be read
         if magic_word_test_bytes_len < magic_word_len:
-            # check for more bytes once every second for up to number of seconds in status beacon period
-            for _ in range(SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS):
+            # check for more bytes once every second for up to four seconds longer than number of seconds in status beacon period # Tanner (3/16/21): issue seen with simulator taking slightly longer than status beacon period to send next data packet
+            for _ in range(SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS + 4):
                 num_bytes_remaining = magic_word_len - magic_word_test_bytes_len
                 next_bytes = board.read(size=num_bytes_remaining)
                 magic_word_test_bytes += next_bytes
@@ -66,14 +213,24 @@ def _register_magic_word(self, board_idx: int) -> None:
                 sleep(1)
             else:
                 # if the entire period has passed and no more bytes are available an error has occured with the Mantarray that is considered fatal
-                raise SerialCommPacketRegistrationTimoutError()
+                raise SerialCommPacketRegistrationTimoutError(magic_word_test_bytes)
+        # read more bytes until the magic word is registered, the timeout value is reached, or the maximum number of bytes are read
         num_bytes_checked = 0
-        while magic_word_test_bytes != SERIAL_COMM_MAGIC_WORD_BYTES:
+        read_dur_secs = 0.0
+        start = perf_counter()
+        while (
+            magic_word_test_bytes != SERIAL_COMM_MAGIC_WORD_BYTES
+            and read_dur_secs < SERIAL_COMM_STATUS_BEACON_PERIOD_SECONDS
+        ):
             next_byte = board.read(size=1)
-            if len(next_byte) == 0:
-                raise SerialCommPacketRegistrationReadEmptyError()
-            magic_word_test_bytes = magic_word_test_bytes[1:] + next_byte
-            num_bytes_checked += 1
-            if num_bytes_checked > SERIAL_COMM_MAX_PACKET_LENGTH_BYTES:
-                raise SerialCommPacketRegistrationSearchExhaustedError()
+            if len(next_byte) == 1:
+                magic_word_test_bytes = magic_word_test_bytes[1:] + next_byte
+                num_bytes_checked += 1
+                # A magic word should be encountered if this many bytes are read. If not, we can assume there was a problem with the mantarray
+                if num_bytes_checked > SERIAL_COMM_MAX_PACKET_LENGTH_BYTES:
+                    raise SerialCommPacketRegistrationSearchExhaustedError()
+            read_dur_secs = _get_seconds_since_read_start(start)
+        # if this point is reached and the magic word has not been found, then at some point no additional bytes were being read
+        if magic_word_test_bytes != SERIAL_COMM_MAGIC_WORD_BYTES:
+            raise SerialCommPacketRegistrationReadEmptyError()
         self._is_registered_with_serial_comm[board_idx] = True