Restructuring of transport protocol code and examples for simplicity. Inclusion of TP class into utils.py

Author: Martin

examples/data-processing/LOG_TP/FCBF0606/00002058/00000001-5FAB8E54.mf4

@@ -16,7 +16,6 @@ Download this folder, enter it, open your command prompt and run below:
 - `process_data.py`: List log files between dates, DBC decode them and perform various processing
 - `process_tp_data.py`: Example of how multiframe data can be handled incl. DBC decoding (Transport Protocol)
 - `utils.py`: Functions/classes used in the above scripts (note: Identical to utils.py from the dashboard-writer repo)
-- `utils_tp.py`: Functions/classes used for Transport Protocol handling
 
 ---
 
@@ -30,15 +29,31 @@ If you're using AWS S3, your endpoint would e.g. be `https://s3.us-east-2.amazon
 ---
 
 ### Regarding Transport Protocol example
-The example in `process_tp_data.py` should be seen as a very simplistic WIP TP implementation. It can be used as a starting point and will most likely need to be modified for individual use cases. We of course welcome any questions/feedback on this functionality.
+The example in `process_tp_data.py` should be seen as a very simplistic TP implementation. It can be used as a starting point and will most likely need to be modified for individual use cases. We of course welcome any questions/feedback on this functionality.
 
 The basic concept works as follows:
 
-1. You specify a list of 'response IDs', which are the CAN IDs with multiframe responses  
-2. The raw data is filtered by the response IDs and the payloads of these frames are combined  
+1. You specify the "type" of transport protocol: UDS (`uds`), J1939 (`j1939`) or NMEA 2000 Fast Packets (`nmea`)
+2. The raw data is filtered by the protocol-specific 'TP response IDs' and the payloads of these frames are combined  
 3. The original response frames are then replaced by these re-constructed frames with payloads >8 bytes  
-4. You can modify how the first/consequtive frames are interpreted (see the UDS and J1939 examples)  
-5. The re-constructed data can be decoded using DBC files, optionally using multiplexing as in the sample UDS DBC files 
+4. The re-constructed data can be decoded using DBC files, optionally using multiplexing as in the sample UDS DBC files 
+
+#### Implementing TP processing in other scripts 
+To use the Transport Protocol functionality in other scripts, you need to make minor modifications:
+
+1. Ensure that you import the `MultiFrameDecoder` class from `utils.py` 
+2. Specify the type via the `tp_type` variable and ensure you include this in the `extract_phys` function 
+
+See below example:
+
+```
+tp_type = "j1939"
+df_raw, device_id = proc.get_raw_data(log_file)
+tp = MultiFrameDecoder(tp_type)
+df_raw = tp.combine_tp_frames(df_raw)
+df_phys = proc.extract_phys(df_raw, tp_type=tp_type)
+```
+
 
 #### UDS example
 For UDS basics see the [Wikipedia article](https://en.wikipedia.org/wiki/Unified_Diagnostic_Services). The UDS example for device `17BD1DB7` shows UDS response data from a Hyunda Kona EV. 
@@ -79,6 +94,3 @@ A UDS DBC file can use extended multiplexing to decode UDS signals, utilizing th
 
 The script merges the reconstructed UDS frames into the original data (removing the original entries of the response ID). The result is a new raw dataframe that can be processed as you would normally do (using a suitable DBC file). The above example has an associated DBC file, `tp_uds_hyundai_soc.dbc`, which lets you extract e.g. State of Charge.
 
------
-### Pending improvements
-- Improve the UDS/J1939 scripts to alternatively trigger the creation of a new combined frame once the payload exceeds the data length 

examples/data-processing/LOG_TP/FCBF0606/00003482/00000001-60B621D9.MF4

@@ -1,20 +1,15 @@
-import mdf_iter, canedge_browser, can_decoder, os
-import pandas as pd
-from datetime import datetime, timezone
-from utils import setup_fs, load_dbc_files, ProcessData
-from utils_tp import MultiFrameDecoder, nmea_fast_packet_pgns
+import canedge_browser, os
+from utils import setup_fs, load_dbc_files, ProcessData, MultiFrameDecoder
 
-# ---------------------------------------------------
-# initialize DBC converter and file loader
-def process_tp_example(devices, dbc_path, res_id_list_hex, tp_type):
+
+def process_tp_example(devices, dbc_path, tp_type):
     fs = setup_fs(s3=False)
     db_list = load_dbc_files(dbc_paths)
     log_files = canedge_browser.get_log_files(fs, devices)
 
     proc = ProcessData(fs, db_list)
 
     for log_file in log_files:
-        # create output folder
         output_folder = "output" + log_file.replace(".MF4", "")
         if not os.path.exists(output_folder):
             os.makedirs(f"{output_folder}")
@@ -23,11 +18,11 @@ def process_tp_example(devices, dbc_path, res_id_list_hex, tp_type):
         df_raw.to_csv(f"{output_folder}/tp_raw_data.csv")
 
         # replace transport protocol sequences with single frames
-        tp = MultiFrameDecoder(df_raw, res_id_list_hex)
-        df_raw = tp.combine_tp_frames_by_type(tp_type)
+        tp = MultiFrameDecoder(tp_type)
+        df_raw = tp.combine_tp_frames(df_raw)
         df_raw.to_csv(f"{output_folder}/tp_raw_data_combined.csv")
 
-        # extract physical values as normal
+        # extract physical values as normal, but add tp_type
         df_phys = proc.extract_phys(df_raw, tp_type=tp_type)
         df_phys.to_csv(f"{output_folder}/tp_physical_values.csv")
 
@@ -37,37 +32,27 @@ def process_tp_example(devices, dbc_path, res_id_list_hex, tp_type):
 # ----------------------------------------
 # run different TP examples
 
-# # basic UDS example with multiple UDS PIDs on same CAN ID, e.g. 221100, 221101
-# devices = ["LOG_TP/0D2C6546"]
-# dbc_paths = [r"dbc_files/tp_uds_test.dbc"]
-# res_id_list_hex = ["0x7E9"]
-#
-# process_tp_example(devices, dbc_paths, res_id_list_hex, "uds")
-#
-# # UDS data from Hyundai Kona EV (SoC%)
+# UDS data from Hyundai Kona EV (SoC%)
 devices = ["LOG_TP/17BD1DB7"]
 dbc_paths = [r"dbc_files/tp_uds_hyundai_soc.dbc"]
-res_id_list_hex = ["0x7EC", "0x7BB"]
+process_tp_example(devices, dbc_paths, "uds")
 
-process_tp_example(devices, dbc_paths, res_id_list_hex, "uds")
+# J1939 TP data
+devices = ["LOG_TP/FCBF0606"]
+dbc_paths = [r"dbc_files/tp_j1939.dbc"]
+process_tp_example(devices, dbc_paths, "j1939")
 
-# # J1939 TP data
-# devices = ["LOG_TP/FCBF0606"]
-# res_id_list_hex = ["0x1CEBFF00"]
-# dbc_paths = [r"dbc_files/tp_j1939.dbc"]
-#
-# process_tp_example(devices, dbc_paths, res_id_list_hex, "j1939")
+# NMEA 2000 fast packet data (with GNSS position)
+devices = ["LOG_TP/94C49784"]
+dbc_paths = [r"dbc_files/tp_nmea_2.dbc"]
+process_tp_example(devices, dbc_paths, "nmea")
 
 # UDS data across two CAN channels
-# devices = ["LOG_TP/FE34E37D"]
-# dbc_paths = [r"dbc_files/tp_uds_test.dbc"]
-# res_id_list_hex = ["0x7EA"]
-#
-# process_tp_example(devices, dbc_paths, res_id_list_hex, "uds")
-
-# NMEA 2000 TP data (with GNSS position)
-# devices = ["LOG_TP/94C49784"]
-# res_id_list_hex = nmea_fast_packet_pgns
-# dbc_paths = [r"dbc_files/tp_nmea_2.dbc"]
-#
-# process_tp_example(devices, dbc_paths, res_id_list_hex, "nmea")
+devices = ["LOG_TP/FE34E37D"]
+dbc_paths = [r"dbc_files/tp_uds_test.dbc"]
+process_tp_example(devices, dbc_paths, "uds")
+
+# UDS example with multiple UDS PIDs on same CAN ID, e.g. 221100, 221101
+devices = ["LOG_TP/0D2C6546"]
+dbc_paths = [r"dbc_files/tp_uds_test.dbc"]
+process_tp_example(devices, dbc_paths, "uds")

examples/data-processing/README.md

@@ -208,3 +208,240 @@ def print_log_summary(self, device_id, log_file, df_phys):
                 "\n---------------",
                 f"\nDevice: {device_id} | Log file: {log_file.split(device_id)[-1]} [Extracted {len(df_phys)} decoded frames]\nPeriod: {df_phys.index.min()} - {df_phys.index.max()}\n",
             )
+
+
+# -----------------------------------------------
+class MultiFrameDecoder:
+    """BETA class for handling transport protocol data. For each response ID, identify
+    sequences of subsequent frames and combine the relevant parts of the data payloads
+    into a single payload with the response ID as the ID. The original raw dataframe is
+    then cleansed of the original response ID sequence frames. Instead, the new concatenated
+    frames are inserted. Further, the class supports DBC decoding of the resulting modified raw data
+
+    :param tp_type:                     the class supports UDS ("uds"), NMEA 2000 Fast Packets ("nmea") and J1939 ("j1939")
+    :param df_raw:                      dataframe of raw CAN data from the mdf_iter module
+
+    SINGLE_FRAME_MASK:                  mask used in matching single frames
+    FIRST_FRAME_MASK:                   mask used in matching first frames
+    CONSEQ_FRAME_MASK:                  mask used in matching consequtive frames
+    SINGLE_FRAME:                       frame type reflecting a single frame response
+    FIRST_FRAME:                        frame type reflecting the first frame in a multi frame response
+    CONSEQ_FRAME:                       frame type reflecting a consequtive frame in a multi frame response
+    ff_payload_start:                   the combined payload will start at this byte in the FIRST_FRAME
+    bam_pgn_hex:                        this is used in J1939 and marks the initial BAM message ID in HEX
+    res_id_list_hex:                    TP 'response CAN IDs' to process. For nmea/j1939, these are provided by default
+
+    """
+
+    def __init__(self, tp_type=""):
+        frame_struct_uds = {
+            "SINGLE_FRAME_MASK": 0xF0,
+            "FIRST_FRAME_MASK": 0xF0,
+            "CONSEQ_FRAME_MASK": 0xF0,
+            "SINGLE_FRAME": 0x00,
+            "FIRST_FRAME": 0x10,
+            "CONSEQ_FRAME": 0x20,
+            "ff_payload_start": 2,
+            "bam_pgn_hex": "",
+            "res_id_list_hex": ["0x7E0", "0x7E9", "0x7EA", "0x7EB", "0x7EC", "0x7ED", "0x7EE", "0x7EF", "0x7EA", "0x7BB"],
+        }
+
+        frame_struct_j1939 = {
+            "SINGLE_FRAME_MASK": 0xFF,
+            "FIRST_FRAME_MASK": 0xFF,
+            "CONSEQ_FRAME_MASK": 0x00,
+            "SINGLE_FRAME": 0xFF,
+            "FIRST_FRAME": 0x20,
+            "CONSEQ_FRAME": 0x00,
+            "ff_payload_start": 8,
+            "bam_pgn_hex": "0xEC00",
+            "res_id_list_hex": ["0xEB00"],
+        }
+
+        frame_struct_nmea = {
+            "SINGLE_FRAME_MASK": 0xFF,
+            "FIRST_FRAME_MASK": 0x0F,
+            "CONSEQ_FRAME_MASK": 0x00,
+            "SINGLE_FRAME": 0xFF,
+            "FIRST_FRAME": 0x00,
+            "CONSEQ_FRAME": 0x00,
+            "ff_payload_start": 2,
+            "bam_pgn_hex": "",
+            "res_id_list_hex": [
+                "0xfed8",
+                "0x1f007",
+                "0x1f008",
+                "0x1f009",
+                "0x1f014",
+                "0x1f016",
+                "0x1f101",
+                "0x1f105",
+                "0x1f201",
+                "0x1f208",
+                "0x1f209",
+                "0x1f20a",
+                "0x1f20c",
+                "0x1f20f",
+                "0x1f210",
+                "0x1f212",
+                "0x1f513",
+                "0x1f805",
+                "0x1f80e",
+                "0x1f80f",
+                "0x1f810",
+                "0x1f811",
+                "0x1f814",
+                "0x1f815",
+                "0x1f904",
+                "0x1f905",
+                "0x1fa04",
+                "0x1fb02",
+                "0x1fb03",
+                "0x1fb04",
+                "0x1fb05",
+                "0x1fb11",
+                "0x1fb12",
+                "0x1fd10",
+                "0x1fe07",
+                "0x1fe12",
+                "0x1ff14",
+                "0x1ff15",
+            ],
+        }
+
+        if tp_type == "uds":
+            self.frame_struct = frame_struct_uds
+        elif tp_type == "j1939":
+            self.frame_struct = frame_struct_j1939
+        elif tp_type == "nmea":
+            self.frame_struct = frame_struct_nmea
+        else:
+            self.frame_struct = {}
+
+        self.tp_type = tp_type
+
+        return
+
+    def calculate_pgn(self, frame_id):
+        pgn = (frame_id & 0x03FFFF00) >> 8
+
+        pgn_f = (pgn & 0xFF00) >> 8
+        pgn_s = pgn & 0x00FF
+
+        if pgn_f < 240:
+            pgn &= 0xFFFFFF00
+
+        return pgn
+
+    def construct_new_tp_frame(self, base_frame, payload_concatenated, can_id):
+        new_frame = base_frame
+        new_frame.at["DataBytes"] = payload_concatenated
+        new_frame.at["DLC"] = 0
+        new_frame.at["DataLength"] = len(payload_concatenated)
+
+        if can_id:
+            new_frame.at["ID"] = can_id
+
+        return new_frame
+
+    def combine_tp_frames(self, df_raw):
+        import pandas as pd
+        import sys
+
+        bam_pgn_hex = self.frame_struct["bam_pgn_hex"]
+        res_id_list = [int(res_id, 16) for res_id in self.frame_struct["res_id_list_hex"]]
+
+        df_raw_combined = pd.DataFrame()
+
+        # use PGN matching for J1939 and NMEA
+        if self.tp_type == "nmea" or self.tp_type == "j1939":
+            df_raw_excl_tp = df_raw[~df_raw["ID"].apply(self.calculate_pgn).isin(res_id_list)]
+        else:
+            df_raw_excl_tp = df_raw[~df_raw["ID"].isin(res_id_list)]
+
+        df_raw_combined = df_raw_excl_tp
+
+        for channel, df_raw_channel in df_raw.groupby("BusChannel"):
+            for res_id in res_id_list:
+                # filter raw data for response ID and extract a 'base frame'
+                if bam_pgn_hex == "":
+                    bam_pgn = 0
+                else:
+                    bam_pgn = int(bam_pgn_hex, 16)
+
+                if self.tp_type == "nmea" or self.tp_type == "j1939":
+                    df_raw_filter = df_raw_channel[df_raw_channel["ID"].apply(self.calculate_pgn).isin([res_id, bam_pgn])]
+                else:
+                    df_raw_filter = df_raw_channel[df_raw_channel["ID"].isin([res_id])]
+
+                if df_raw_filter.empty:
+                    continue
+
+                base_frame = df_raw_filter.iloc[0]
+
+                frame_list = []
+                frame_timestamp_list = []
+                payload_concatenated = []
+                ff_length = 0xFFF
+                can_id = None
+                conseq_frame_prev = None
+
+                # iterate through rows in filtered dataframe
+                for index, row in df_raw_filter.iterrows():
+                    payload = row["DataBytes"]
+                    first_byte = payload[0]
+                    row_id = row["ID"]
+                    row_pgn = self.calculate_pgn(row_id)
+
+                    # check if first frame (either for UDS/NMEA or J1939 case)
+                    first_frame_test = (
+                        (first_byte & self.frame_struct["FIRST_FRAME_MASK"] == self.frame_struct["FIRST_FRAME"])
+                        & (bam_pgn_hex == "")
+                    ) or (self.tp_type == "j1939" and bam_pgn == row_pgn)
+
+                    # if single frame, save frame directly (excl. 1st byte)
+                    if first_byte & self.frame_struct["SINGLE_FRAME_MASK"] == self.frame_struct["SINGLE_FRAME"]:
+                        new_frame = self.construct_new_tp_frame(base_frame, payload, row_id)
+                        frame_list.append(new_frame.values.tolist())
+                        frame_timestamp_list.append(index)
+
+                    # if first frame, save info from prior multi frame response sequence,
+                    # then initialize a new sequence incl. the first frame payload
+                    elif first_frame_test:
+                        # create a new frame using information from previous iterations
+                        if len(payload_concatenated) >= ff_length:
+                            new_frame = self.construct_new_tp_frame(base_frame, payload_concatenated, can_id)
+
+                            frame_list.append(new_frame.values.tolist())
+                            frame_timestamp_list.append(frame_timestamp)
+
+                        # reset and start on next frame
+                        payload_concatenated = []
+                        conseq_frame_prev = None
+                        frame_timestamp = index
+
+                        # for J1939, extract PGN and convert to 29 bit CAN ID for use in baseframe
+                        if self.tp_type == "j1939":
+                            pgn_hex = "".join("{:02x}".format(x) for x in reversed(payload[5:8]))
+                            pgn = int(pgn_hex, 16)
+                            can_id = (6 << 26) | (pgn << 8) | 254
+
+                        ff_length = (payload[0] & 0x0F) << 8 | payload[1]
+
+                        for byte in payload[self.frame_struct["ff_payload_start"] :]:
+                            payload_concatenated.append(byte)
+
+                    # if consequtive frame, extend payload with payload excl. 1st byte
+                    elif first_byte & self.frame_struct["CONSEQ_FRAME_MASK"] == self.frame_struct["CONSEQ_FRAME"]:
+                        if (conseq_frame_prev == None) or ((first_byte - conseq_frame_prev) == 1):
+                            conseq_frame_prev = first_byte
+                            for byte in payload[1:]:
+                                payload_concatenated.append(byte)
+
+                df_raw_tp = pd.DataFrame(frame_list, columns=base_frame.index, index=frame_timestamp_list)
+                df_raw_combined = df_raw_combined.append(df_raw_tp)
+
+        df_raw_combined.index.name = "TimeStamp"
+        df_raw_combined = df_raw_combined.sort_index()
+
+        return df_raw_combined