From 392726a90656000a02db1b7983c4bb3235877ccc Mon Sep 17 00:00:00 2001 From: Joshua Lucas Date: Wed, 14 Oct 2026 21:23:21 -0500 Subject: [PATCH] Finish address offset calculation for XSY module. --- .../assets/flatten_datablock.xslt | 4 +- .../assets/flatten_ddtsource.xslt | 4 +- src/AuDaGe/formats/controlexpert/xsy.py | 251 +++++++++++------- tests/test_xsy.py | 15 +- 4 files changed, 171 insertions(+), 103 deletions(-) diff --git a/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt b/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt index d1fd310..a75e5d9 100644 --- a/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt +++ b/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt @@ -19,12 +19,12 @@ -
+ + - :
diff --git a/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt b/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt index 093107a..5466d5f 100644 --- a/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt +++ b/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt @@ -24,9 +24,10 @@ -
+ : + @@ -35,7 +36,6 @@ - : diff --git a/src/AuDaGe/formats/controlexpert/xsy.py b/src/AuDaGe/formats/controlexpert/xsy.py index 51fee68..566f189 100644 --- a/src/AuDaGe/formats/controlexpert/xsy.py +++ b/src/AuDaGe/formats/controlexpert/xsy.py @@ -15,13 +15,14 @@ from collections.abc import Iterable from dataclasses import dataclass from enum import IntEnum, StrEnum, auto from importlib import resources +from itertools import pairwise import pandas as pd +from numpy import nan -from AuDaGe import iec -from AuDaGe.audage_frame import AuDaGeFrame +from AuDaGe import frame, iec +from AuDaGe.better_enum import BetterIntEnum from AuDaGe.debug_tools import MeasureTime, TimeUnit -from AuDaGe.format import Format ASSETS_PATH = "AuDaGe.formats.controlexpert.assets" @@ -31,23 +32,19 @@ MAX_RECURSION = 5 module_logger = logging.getLogger(__name__) -class XSYBlock(AuDaGeFrame): +class XSYBlock(frame.AuDaGeFrame): """Base class for DataBlock and Definitions that handles shared attributes. May be instantiated directly from a dataframe or from an XSY file using the from_file class method. """ - # _stylesheet_path: str # = None - # _xsy_path: str # = None - _name: str - @classmethod def from_file( cls, xsy_path: os.PathLike, stylesheet_path: os.PathLike, name_ext: str = "", - ) -> XSYBlock: # noqa: F821 + ) -> XSYBlock: """Load an XSY file into a dataframe using the specified stylesheet.""" cls.stylesheet_path = stylesheet_path cls.xsy_path = xsy_path @@ -55,7 +52,6 @@ class XSYBlock(AuDaGeFrame): resources.as_file(cls.xsy_path) as xsy_file, resources.as_file(cls.stylesheet_path) as stylesheet_file, ): - # frame = AuDaGeFrame(pd.read_xml(xsy_file, stylesheet=stylesheet_file)) return cls( pd.read_xml(xsy_file, stylesheet=stylesheet_file), ".".join( @@ -63,6 +59,26 @@ class XSYBlock(AuDaGeFrame): ), ) + def _make_array_definitions( + self, frame: frame.AuDaGeFrame | None = None + ) -> frame.AuDaGeFrame | None: + """Make definitions for the expansion of arrays of UDTs.""" + frame = self if frame is None else frame + frame_grouped = frame.groupby("type") + array_definitions_list = [] + for name, group in frame_grouped: + try: + array = iec.Array(name) + if not array.is_elementary(): + array_definitions_list.append(array.make_definition("")) + except iec.InvalidSpecError: + continue + try: + return pd.concat(array_definitions_list) + # Return `None` if there is nothing to concatenate. + except ValueError: + return None + class DataBlock(XSYBlock): """Read and parse the dataBlock node of an XSY file.""" @@ -74,11 +90,11 @@ class DataBlock(XSYBlock): @classmethod def from_file( cls, xsy_path: os.PathLike, stylesheet_path: os.PathLike | None = None - ) -> DataBlock: # noqa: F821 + ) -> DataBlock: """Read the dataBlock of an XSY file into a DataFrame.""" if stylesheet_path is None: stylesheet_path = cls.DEFAULT_SS_PATH - return cls(XSYBlock.from_file(xsy_path, stylesheet_path, "DataBlock")) + return cls(super().from_file(xsy_path, stylesheet_path, "DataBlock")) class Definitions(XSYBlock): @@ -91,6 +107,14 @@ class Definitions(XSYBlock): "builtins.xsy" ) + class BoolParents(BetterIntEnum): + """Enum of UDTs that pack booleans into data blocks other than bits or + words. Each member name is the name of the UDT and the value is the + number of bits that each boolean is packed into. + """ + + PARA_PI_B = 8 + @classmethod def from_file( cls, @@ -98,7 +122,7 @@ class Definitions(XSYBlock): stylesheet_path: os.path | None = None, definition_paths: Iterable[os.path] | None = None, include_builtins: bool = True, - ) -> Definitions: # noqa: F821 + ) -> Definitions: """Read the DDTSource nodes of an XSY file into a DataFrame.""" cls.stylesheet_path = ( cls.DEFAULT_SS_PATH if stylesheet_path is None else stylesheet_path @@ -122,38 +146,71 @@ class Definitions(XSYBlock): _name = ".".join( (os.path.splitext(os.path.basename(xsy_path))[0], "Definitions") ) - with MeasureTime( - TimeUnit.MS, - True, + definitions = XSYBlock(pd.concat(cls.definitions_list)) + definitions_extended = pd.concat( + (definitions, definitions._make_array_definitions()) + ) + time_message = ( "{_name} offset calculation completed in {time} {unit}.".format( _name=_name, time="{time:.3f}", unit="{unit}" - ), - ): - offsets = cls._calculate_offsets( - pd.concat(cls.definitions_list), - _name=_name, ) + ) + with MeasureTime(TimeUnit.MS, True, time_message): + offsets = cls._calculate_offsets(definitions_extended, _name=_name) return cls(offsets) - def _get_length(self, row, udt_lengths: dict[str, int]) -> int: - # Find packed-bit words by checking whether... + @classmethod + def _get_length( + cls, + row, + udt_lengths: dict[str, int], + bool_parents: IntEnum = BoolParents, + ) -> int: + # Special case to fix the BOOLs that are packed into data units other + # than bits or words. if ( - # ...the first member is a 16-bit binary or integer type... - row["type"] in ("WORD", "INT", "UINT") - # ...and that the second member is a boolean... - and row["next_type"] == "BOOL" - # ...and that the second member has a bitrank. - and row["next_bitrank"].notna() + bool_parents.contains(row["parent"].upper()) + and row["type"] == "BOOL" ): - # We don't increase the offset for packed-bit words because the - # following booleans overlap with them. + return bool_parents[row["parent"].upper()].value + # Bit- and byte- length variables that are packed into words have + # functionally zero length for our purposes. + if row["type"] in ( + "BOOL", + "EBOOL", + "BYTE", + "SINT", + "USINT", + "CHAR", + ) and pd.notna(row["bitrank"]): return 0 + length = iec.get_data_length( + row["type"], + udt_lengths=udt_lengths, + unknown_length=nan, + ) + if pd.isna(length): + return length else: - return iec.get_data_length( - row["type"], - udt_lengths=udt_lengths, - unknown_length=pd.NA, - ) + return max(length, 16) + + @classmethod + def _calculate_padding( + cls, + address: int | nan, + length: int | nan, + max_alignment: int = iec.DataUnit.DWORD, + ) -> int | nan: + # Return early if missing information. + if pd.isna(address) or pd.isna(length) or length <= 0: + return address + alignment = min(length, max_alignment) + rem = address % alignment + # No change is necessary if address is divisible by length. + if rem <= 0: + return address + # Increase address to make it divisible by length. + return address + alignment - rem @classmethod def _calculate_offsets( @@ -163,7 +220,7 @@ class Definitions(XSYBlock): max_recursion: int = MAX_RECURSION, _current_recursion: int = 0, _name: str = "", - ) -> Definitions: # noqa: F821 + ) -> Definitions: """Recursively calculate the memory offset for all UDT members in the definitions. """ @@ -183,30 +240,11 @@ class Definitions(XSYBlock): udt_lengths = {} if udt_lengths is None else udt_lengths # We don't care about DFBs so set their length at zero. udt_lengths["DFB"] = 0 - definitions_joined = definitions.shift((0, -1), axis=0) - definitions_joined.to_csv("definitions_joined.csv") # Create a column with the length of each member. - definitions["length"] = definitions_joined.apply( - lambda row: ( - 0 - if ( - row["type_0"] in ("WORD", "INT", "UINT") - and row["type_-1"] == "BOOL" - and pd.notna(row["bitrank_-1"]) - ) - else iec.get_data_length( - row["type_0"], - udt_lengths=udt_lengths, - unknown_length=pd.NA, - ) - ), + definitions["length"] = definitions.apply( + lambda row: cls._get_length(row, udt_lengths), axis=1, - ).shift(1, axis=0, fill_value=0) - definitions.to_csv("definitions.csv") - # Calculate how many bits need to be skipped before each member so - # that addressing rules are followed (e.g. word-length types on - # integer addresses, multi-word types on even addresses). - # definitions["padding"] = definitions.apply() + ) # Set a flag high for checking whether all offsets have been found. # If this isn't set False by anything later then we're done. all_offsets_found = True @@ -214,16 +252,24 @@ class Definitions(XSYBlock): # definition. This is the offset of the subsequent member. definitions_calculated_list = [] for name, group in definitions.groupby("parent"): - group["offset"] = group["length"].cumsum(skipna=False) + group_dict = group.to_dict("records") + group_dict[0]["address"] = 0 + group_dict_calc = [group_dict[0]] + for prev_row, row in pairwise(group_dict): + row["address"] = cls._calculate_padding( + prev_row["address"] + prev_row["length"], row["length"] + ) + group_dict_calc.append(row) + group_calc = frame.AuDaGeFrame(group_dict_calc) # Check if all offsets for this UDT has been found. If so, we # know its length so we save it to the `udt_lengths` dictionary # for future reference. - if group["offset"].notna().all(): - udt_lengths[name] = group["offset"].tail(1).values[0] + if group_calc["address"].notna().all(): + udt_lengths[name] = group_calc["address"].tail(1).values[0] else: # This UDT's offsets aren't finished so set the flag False. all_offsets_found = False - definitions_calculated_list.append(group) + definitions_calculated_list.append(group_calc) # Re-combine all the UDT definitions. definitions_calculated = pd.concat(definitions_calculated_list) if all_offsets_found: @@ -269,7 +315,7 @@ class XSY: GROUP = auto() # 2 JOIN = auto() # 3 COMPILE = auto() # 4 - NONE = auto() # 5 + COMPLETE = auto() # 5 class IgnoredDataTypes(StrEnum): DFB = auto() @@ -279,7 +325,7 @@ class XSY: datablock: DataBlock | None = None, definitions: Definitions | None = None, ignored_data_types: StrEnum = IgnoredDataTypes, - debug_step: DebugStep | int = DebugStep.NONE, + debug_step: DebugStep | int = DebugStep.COMPLETE, max_recursion: int = MAX_RECURSION, ) -> DataBlock: """Recursively expand the UDTs in `datablock` based on the UDT @@ -321,11 +367,8 @@ class XSY: unit="{unit}", ), ): - array_definitions = self._make_array_definitions( - datablock, definitions - ) definitions_extended = pd.concat( - (definitions, array_definitions) + (definitions, datablock._make_array_definitions()) ) if debug_step == self.DebugStep.MAKE_ARRAY_DEFS: return definitions_extended @@ -338,6 +381,33 @@ class XSY: ) return datablock_expanded + def _combine_addresses(self, row: pd.Series) -> iec.Address: + if pd.isna(row["address"]): + return nan + address_object = iec.Address.from_string(row["address"]) + if pd.notna(row["address_def"]): + offset = row["address_def"] // 16 + else: + offset = 0 + if pd.notna(row["bitrank_def"]): + bit = float(row["bitrank_def"]) + if address_object.bit is None: + address_object.bit = bit + else: + address_object.bit += bit + offset -= 1 + address_object.word += offset + return str(address_object) + + def _combine_comments(self, *comments: str) -> str: + comments_cleaned = [] + for comment in comments: + if pd.isna(comment) or comment == "": + continue + comment_cleaned = comment.lstrip(": ") + comments_cleaned.append(comment_cleaned) + return ": ".join(comments_cleaned) + def _compile_datablock(self, datablock: DataBlock) -> DataBlock: """Compile a cross-joined and filtered DataBlock by combining the columns from the datablock with the columns from the definitions.""" @@ -347,11 +417,17 @@ class XSY: ) datablock_compiled["parent"] = "" datablock_compiled["type"] = datablock["type_def"] - datablock_compiled["address"] = datablock["address"] + datablock_compiled["address"] = datablock.apply( + lambda row: self._combine_addresses(row), + axis=1, + ) datablock_compiled["bitrank"] = datablock["bitrank_def"] datablock_compiled["save"] = datablock["save"] datablock_compiled["comment"] = datablock.apply( - lambda row: f"{row['comment']}: {row['comment_def']}", axis=1 + lambda row: self._combine_comments( + row["comment"], row["comment_def"] + ), + axis=1, ) return DataBlock(datablock_compiled) @@ -454,23 +530,8 @@ class XSY: ) return pd.concat((edts, udts_finished)) - def _make_array_definitions( - self, datablock: DataBlock, definitions: Definitions - ) -> Definitions: - """Make definitions for the expansion of arrays of UDTs.""" - everything_grouped = pd.concat((datablock, definitions)).groupby("type") - array_definitions_list = [] - for name, group in everything_grouped: - try: - array = iec.Array(name) - if not array.is_elementary(): - array_definitions_list.append(array.make_definition("")) - except iec.InvalidSpecError: - continue - return pd.concat(array_definitions_list) - -class XSYReader(Format): +class XSYReader(frame.Reader): """Class for reading and writing Schneider Electric EcoStruxure Control Expert variable exchange (XSY) files. Can be re-used to read multiple XSY files using common stylesheets and UDT definitions. @@ -483,7 +544,8 @@ class XSYReader(Format): definitions_ss_path: os.path | None = None, even_addresses: bool = True, ): - """Initialize XSYReader object. + """Initialize XSYReader object with default definition files and + stylesheets. Parameters ---------- @@ -510,16 +572,19 @@ class XSYReader(Format): """Read the dataBlock and definitions of an XSY file into an XSY object. """ - self.datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path) - self.definitions = Definitions.from_file( + datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path) + definitions = Definitions.from_file( xsy_path, self.definitions_ss_path, self.definition_paths ) - return XSY( - datablock=self.datablock, - definitions=self.definitions, + xsy = XSY( + datablock=datablock, + definitions=definitions, even_addresses=self.even_addresses, name=os.path.basename(xsy_path), ) + if expand: + xsy.datablock = xsy.expand_datablock() + return xsy def write(self): """Write XSY object to file.""" diff --git a/tests/test_xsy.py b/tests/test_xsy.py index 5312a9b..b2d90d6 100644 --- a/tests/test_xsy.py +++ b/tests/test_xsy.py @@ -9,16 +9,17 @@ desc: Unit tests for the AuDaGe XSY module. """ import os - -# import time import unittest from importlib import resources -import output - -import assets from AuDaGe.formats.controlexpert.xsy import * +try: + from tests import assets, output +except ImportError: # pragma: no cover + import assets # noqa: I001 + import output + class TestXSY(unittest.TestCase): """This test case tests the XSYBlock, DataBlock, and Definitions classes in @@ -76,7 +77,9 @@ class TestXSYReader(unittest.TestCase): index=False, ) debug = False - for debug_step in XSY.DebugStep if debug else [XSY.DebugStep.NONE]: + for debug_step in ( + XSY.DebugStep if debug else [XSY.DebugStep.COMPLETE] + ): xsy.expand_datablock(debug_step=debug_step).to_csv( self.out_path.joinpath( f"{file_name}_debug_{debug_step.name}.csv"