Finish address offset calculation for XSY module.

This commit is contained in:
Joshua Lucas
2026-10-14 21:23:21 -05:00
parent 0aaa9492b9
commit 392726a906
4 changed files with 171 additions and 103 deletions

View File

@@ -19,12 +19,12 @@
<xsl:template match="variables"> <xsl:template match="variables">
<xsl:copy> <xsl:copy>
<name><xsl:value-of select="@name"/></name> <name><xsl:value-of select="@name"/></name>
<parent/>
<type><xsl:value-of select="@typeName"/></type> <type><xsl:value-of select="@typeName"/></type>
<address><xsl:value-of select="@topologicalAddress"/></address> <address><xsl:value-of select="@topologicalAddress"/></address>
<comment><xsl:value-of select="comment"/></comment>
<parent/>
<bitrank/> <bitrank/>
<save><xsl:value-of select="attribute/@Save"/></save> <save><xsl:value-of select="attribute/@Save"/></save>
<comment><xsl:value-of select="../../comment"/>: <xsl:value-of select="comment"/></comment>
</xsl:copy> </xsl:copy>
</xsl:template> </xsl:template>

View File

@@ -24,9 +24,10 @@
<xsl:template match="structure/variables"> <xsl:template match="structure/variables">
<xsl:copy> <xsl:copy>
<name><xsl:value-of select="@name"/></name> <name><xsl:value-of select="@name"/></name>
<parent><xsl:value-of select="translate(../../@DDTName, $lowercase, $uppercase)"/></parent>
<type><xsl:value-of select="translate(@typeName, $lowercase, $uppercase)"/></type> <type><xsl:value-of select="translate(@typeName, $lowercase, $uppercase)"/></type>
<address/> <address/>
<comment><xsl:value-of select="../../comment"/>: <xsl:value-of select="comment"/></comment>
<parent><xsl:value-of select="translate(../../@DDTName, $lowercase, $uppercase)"/></parent>
<bitrank> <bitrank>
<xsl:for-each select="attribute"> <xsl:for-each select="attribute">
<xsl:if test="@name='ExtractBit'"> <xsl:if test="@name='ExtractBit'">
@@ -35,7 +36,6 @@
</xsl:for-each> </xsl:for-each>
</bitrank> </bitrank>
<save/> <save/>
<comment><xsl:value-of select="../../comment"/>: <xsl:value-of select="comment"/></comment>
</xsl:copy> </xsl:copy>
</xsl:template> </xsl:template>

View File

@@ -15,13 +15,14 @@ from collections.abc import Iterable
from dataclasses import dataclass from dataclasses import dataclass
from enum import IntEnum, StrEnum, auto from enum import IntEnum, StrEnum, auto
from importlib import resources from importlib import resources
from itertools import pairwise
import pandas as pd import pandas as pd
from numpy import nan
from AuDaGe import iec from AuDaGe import frame, iec
from AuDaGe.audage_frame import AuDaGeFrame from AuDaGe.better_enum import BetterIntEnum
from AuDaGe.debug_tools import MeasureTime, TimeUnit from AuDaGe.debug_tools import MeasureTime, TimeUnit
from AuDaGe.format import Format
ASSETS_PATH = "AuDaGe.formats.controlexpert.assets" ASSETS_PATH = "AuDaGe.formats.controlexpert.assets"
@@ -31,23 +32,19 @@ MAX_RECURSION = 5
module_logger = logging.getLogger(__name__) module_logger = logging.getLogger(__name__)
class XSYBlock(AuDaGeFrame): class XSYBlock(frame.AuDaGeFrame):
"""Base class for DataBlock and Definitions that handles shared attributes. """Base class for DataBlock and Definitions that handles shared attributes.
May be instantiated directly from a dataframe or from an XSY file using the May be instantiated directly from a dataframe or from an XSY file using the
from_file class method. from_file class method.
""" """
# _stylesheet_path: str # = None
# _xsy_path: str # = None
_name: str
@classmethod @classmethod
def from_file( def from_file(
cls, cls,
xsy_path: os.PathLike, xsy_path: os.PathLike,
stylesheet_path: os.PathLike, stylesheet_path: os.PathLike,
name_ext: str = "", name_ext: str = "",
) -> XSYBlock: # noqa: F821 ) -> XSYBlock:
"""Load an XSY file into a dataframe using the specified stylesheet.""" """Load an XSY file into a dataframe using the specified stylesheet."""
cls.stylesheet_path = stylesheet_path cls.stylesheet_path = stylesheet_path
cls.xsy_path = xsy_path cls.xsy_path = xsy_path
@@ -55,7 +52,6 @@ class XSYBlock(AuDaGeFrame):
resources.as_file(cls.xsy_path) as xsy_file, resources.as_file(cls.xsy_path) as xsy_file,
resources.as_file(cls.stylesheet_path) as stylesheet_file, resources.as_file(cls.stylesheet_path) as stylesheet_file,
): ):
# frame = AuDaGeFrame(pd.read_xml(xsy_file, stylesheet=stylesheet_file))
return cls( return cls(
pd.read_xml(xsy_file, stylesheet=stylesheet_file), pd.read_xml(xsy_file, stylesheet=stylesheet_file),
".".join( ".".join(
@@ -63,6 +59,26 @@ class XSYBlock(AuDaGeFrame):
), ),
) )
def _make_array_definitions(
self, frame: frame.AuDaGeFrame | None = None
) -> frame.AuDaGeFrame | None:
"""Make definitions for the expansion of arrays of UDTs."""
frame = self if frame is None else frame
frame_grouped = frame.groupby("type")
array_definitions_list = []
for name, group in frame_grouped:
try:
array = iec.Array(name)
if not array.is_elementary():
array_definitions_list.append(array.make_definition(""))
except iec.InvalidSpecError:
continue
try:
return pd.concat(array_definitions_list)
# Return `None` if there is nothing to concatenate.
except ValueError:
return None
class DataBlock(XSYBlock): class DataBlock(XSYBlock):
"""Read and parse the dataBlock node of an XSY file.""" """Read and parse the dataBlock node of an XSY file."""
@@ -74,11 +90,11 @@ class DataBlock(XSYBlock):
@classmethod @classmethod
def from_file( def from_file(
cls, xsy_path: os.PathLike, stylesheet_path: os.PathLike | None = None cls, xsy_path: os.PathLike, stylesheet_path: os.PathLike | None = None
) -> DataBlock: # noqa: F821 ) -> DataBlock:
"""Read the dataBlock of an XSY file into a DataFrame.""" """Read the dataBlock of an XSY file into a DataFrame."""
if stylesheet_path is None: if stylesheet_path is None:
stylesheet_path = cls.DEFAULT_SS_PATH stylesheet_path = cls.DEFAULT_SS_PATH
return cls(XSYBlock.from_file(xsy_path, stylesheet_path, "DataBlock")) return cls(super().from_file(xsy_path, stylesheet_path, "DataBlock"))
class Definitions(XSYBlock): class Definitions(XSYBlock):
@@ -91,6 +107,14 @@ class Definitions(XSYBlock):
"builtins.xsy" "builtins.xsy"
) )
class BoolParents(BetterIntEnum):
"""Enum of UDTs that pack booleans into data blocks other than bits or
words. Each member name is the name of the UDT and the value is the
number of bits that each boolean is packed into.
"""
PARA_PI_B = 8
@classmethod @classmethod
def from_file( def from_file(
cls, cls,
@@ -98,7 +122,7 @@ class Definitions(XSYBlock):
stylesheet_path: os.path | None = None, stylesheet_path: os.path | None = None,
definition_paths: Iterable[os.path] | None = None, definition_paths: Iterable[os.path] | None = None,
include_builtins: bool = True, include_builtins: bool = True,
) -> Definitions: # noqa: F821 ) -> Definitions:
"""Read the DDTSource nodes of an XSY file into a DataFrame.""" """Read the DDTSource nodes of an XSY file into a DataFrame."""
cls.stylesheet_path = ( cls.stylesheet_path = (
cls.DEFAULT_SS_PATH if stylesheet_path is None else stylesheet_path cls.DEFAULT_SS_PATH if stylesheet_path is None else stylesheet_path
@@ -122,38 +146,71 @@ class Definitions(XSYBlock):
_name = ".".join( _name = ".".join(
(os.path.splitext(os.path.basename(xsy_path))[0], "Definitions") (os.path.splitext(os.path.basename(xsy_path))[0], "Definitions")
) )
with MeasureTime( definitions = XSYBlock(pd.concat(cls.definitions_list))
TimeUnit.MS, definitions_extended = pd.concat(
True, (definitions, definitions._make_array_definitions())
)
time_message = (
"{_name} offset calculation completed in {time} {unit}.".format( "{_name} offset calculation completed in {time} {unit}.".format(
_name=_name, time="{time:.3f}", unit="{unit}" _name=_name, time="{time:.3f}", unit="{unit}"
),
):
offsets = cls._calculate_offsets(
pd.concat(cls.definitions_list),
_name=_name,
) )
)
with MeasureTime(TimeUnit.MS, True, time_message):
offsets = cls._calculate_offsets(definitions_extended, _name=_name)
return cls(offsets) return cls(offsets)
def _get_length(self, row, udt_lengths: dict[str, int]) -> int: @classmethod
# Find packed-bit words by checking whether... def _get_length(
cls,
row,
udt_lengths: dict[str, int],
bool_parents: IntEnum = BoolParents,
) -> int:
# Special case to fix the BOOLs that are packed into data units other
# than bits or words.
if ( if (
# ...the first member is a 16-bit binary or integer type... bool_parents.contains(row["parent"].upper())
row["type"] in ("WORD", "INT", "UINT") and row["type"] == "BOOL"
# ...and that the second member is a boolean...
and row["next_type"] == "BOOL"
# ...and that the second member has a bitrank.
and row["next_bitrank"].notna()
): ):
# We don't increase the offset for packed-bit words because the return bool_parents[row["parent"].upper()].value
# following booleans overlap with them. # Bit- and byte- length variables that are packed into words have
# functionally zero length for our purposes.
if row["type"] in (
"BOOL",
"EBOOL",
"BYTE",
"SINT",
"USINT",
"CHAR",
) and pd.notna(row["bitrank"]):
return 0 return 0
length = iec.get_data_length(
row["type"],
udt_lengths=udt_lengths,
unknown_length=nan,
)
if pd.isna(length):
return length
else: else:
return iec.get_data_length( return max(length, 16)
row["type"],
udt_lengths=udt_lengths, @classmethod
unknown_length=pd.NA, def _calculate_padding(
) cls,
address: int | nan,
length: int | nan,
max_alignment: int = iec.DataUnit.DWORD,
) -> int | nan:
# Return early if missing information.
if pd.isna(address) or pd.isna(length) or length <= 0:
return address
alignment = min(length, max_alignment)
rem = address % alignment
# No change is necessary if address is divisible by length.
if rem <= 0:
return address
# Increase address to make it divisible by length.
return address + alignment - rem
@classmethod @classmethod
def _calculate_offsets( def _calculate_offsets(
@@ -163,7 +220,7 @@ class Definitions(XSYBlock):
max_recursion: int = MAX_RECURSION, max_recursion: int = MAX_RECURSION,
_current_recursion: int = 0, _current_recursion: int = 0,
_name: str = "", _name: str = "",
) -> Definitions: # noqa: F821 ) -> Definitions:
"""Recursively calculate the memory offset for all UDT members in the """Recursively calculate the memory offset for all UDT members in the
definitions. definitions.
""" """
@@ -183,30 +240,11 @@ class Definitions(XSYBlock):
udt_lengths = {} if udt_lengths is None else udt_lengths udt_lengths = {} if udt_lengths is None else udt_lengths
# We don't care about DFBs so set their length at zero. # We don't care about DFBs so set their length at zero.
udt_lengths["DFB"] = 0 udt_lengths["DFB"] = 0
definitions_joined = definitions.shift((0, -1), axis=0)
definitions_joined.to_csv("definitions_joined.csv")
# Create a column with the length of each member. # Create a column with the length of each member.
definitions["length"] = definitions_joined.apply( definitions["length"] = definitions.apply(
lambda row: ( lambda row: cls._get_length(row, udt_lengths),
0
if (
row["type_0"] in ("WORD", "INT", "UINT")
and row["type_-1"] == "BOOL"
and pd.notna(row["bitrank_-1"])
)
else iec.get_data_length(
row["type_0"],
udt_lengths=udt_lengths,
unknown_length=pd.NA,
)
),
axis=1, axis=1,
).shift(1, axis=0, fill_value=0) )
definitions.to_csv("definitions.csv")
# Calculate how many bits need to be skipped before each member so
# that addressing rules are followed (e.g. word-length types on
# integer addresses, multi-word types on even addresses).
# definitions["padding"] = definitions.apply()
# Set a flag high for checking whether all offsets have been found. # Set a flag high for checking whether all offsets have been found.
# If this isn't set False by anything later then we're done. # If this isn't set False by anything later then we're done.
all_offsets_found = True all_offsets_found = True
@@ -214,16 +252,24 @@ class Definitions(XSYBlock):
# definition. This is the offset of the subsequent member. # definition. This is the offset of the subsequent member.
definitions_calculated_list = [] definitions_calculated_list = []
for name, group in definitions.groupby("parent"): for name, group in definitions.groupby("parent"):
group["offset"] = group["length"].cumsum(skipna=False) group_dict = group.to_dict("records")
group_dict[0]["address"] = 0
group_dict_calc = [group_dict[0]]
for prev_row, row in pairwise(group_dict):
row["address"] = cls._calculate_padding(
prev_row["address"] + prev_row["length"], row["length"]
)
group_dict_calc.append(row)
group_calc = frame.AuDaGeFrame(group_dict_calc)
# Check if all offsets for this UDT has been found. If so, we # Check if all offsets for this UDT has been found. If so, we
# know its length so we save it to the `udt_lengths` dictionary # know its length so we save it to the `udt_lengths` dictionary
# for future reference. # for future reference.
if group["offset"].notna().all(): if group_calc["address"].notna().all():
udt_lengths[name] = group["offset"].tail(1).values[0] udt_lengths[name] = group_calc["address"].tail(1).values[0]
else: else:
# This UDT's offsets aren't finished so set the flag False. # This UDT's offsets aren't finished so set the flag False.
all_offsets_found = False all_offsets_found = False
definitions_calculated_list.append(group) definitions_calculated_list.append(group_calc)
# Re-combine all the UDT definitions. # Re-combine all the UDT definitions.
definitions_calculated = pd.concat(definitions_calculated_list) definitions_calculated = pd.concat(definitions_calculated_list)
if all_offsets_found: if all_offsets_found:
@@ -269,7 +315,7 @@ class XSY:
GROUP = auto() # 2 GROUP = auto() # 2
JOIN = auto() # 3 JOIN = auto() # 3
COMPILE = auto() # 4 COMPILE = auto() # 4
NONE = auto() # 5 COMPLETE = auto() # 5
class IgnoredDataTypes(StrEnum): class IgnoredDataTypes(StrEnum):
DFB = auto() DFB = auto()
@@ -279,7 +325,7 @@ class XSY:
datablock: DataBlock | None = None, datablock: DataBlock | None = None,
definitions: Definitions | None = None, definitions: Definitions | None = None,
ignored_data_types: StrEnum = IgnoredDataTypes, ignored_data_types: StrEnum = IgnoredDataTypes,
debug_step: DebugStep | int = DebugStep.NONE, debug_step: DebugStep | int = DebugStep.COMPLETE,
max_recursion: int = MAX_RECURSION, max_recursion: int = MAX_RECURSION,
) -> DataBlock: ) -> DataBlock:
"""Recursively expand the UDTs in `datablock` based on the UDT """Recursively expand the UDTs in `datablock` based on the UDT
@@ -321,11 +367,8 @@ class XSY:
unit="{unit}", unit="{unit}",
), ),
): ):
array_definitions = self._make_array_definitions(
datablock, definitions
)
definitions_extended = pd.concat( definitions_extended = pd.concat(
(definitions, array_definitions) (definitions, datablock._make_array_definitions())
) )
if debug_step == self.DebugStep.MAKE_ARRAY_DEFS: if debug_step == self.DebugStep.MAKE_ARRAY_DEFS:
return definitions_extended return definitions_extended
@@ -338,6 +381,33 @@ class XSY:
) )
return datablock_expanded return datablock_expanded
def _combine_addresses(self, row: pd.Series) -> iec.Address:
if pd.isna(row["address"]):
return nan
address_object = iec.Address.from_string(row["address"])
if pd.notna(row["address_def"]):
offset = row["address_def"] // 16
else:
offset = 0
if pd.notna(row["bitrank_def"]):
bit = float(row["bitrank_def"])
if address_object.bit is None:
address_object.bit = bit
else:
address_object.bit += bit
offset -= 1
address_object.word += offset
return str(address_object)
def _combine_comments(self, *comments: str) -> str:
comments_cleaned = []
for comment in comments:
if pd.isna(comment) or comment == "":
continue
comment_cleaned = comment.lstrip(": ")
comments_cleaned.append(comment_cleaned)
return ": ".join(comments_cleaned)
def _compile_datablock(self, datablock: DataBlock) -> DataBlock: def _compile_datablock(self, datablock: DataBlock) -> DataBlock:
"""Compile a cross-joined and filtered DataBlock by combining the """Compile a cross-joined and filtered DataBlock by combining the
columns from the datablock with the columns from the definitions.""" columns from the datablock with the columns from the definitions."""
@@ -347,11 +417,17 @@ class XSY:
) )
datablock_compiled["parent"] = "" datablock_compiled["parent"] = ""
datablock_compiled["type"] = datablock["type_def"] datablock_compiled["type"] = datablock["type_def"]
datablock_compiled["address"] = datablock["address"] datablock_compiled["address"] = datablock.apply(
lambda row: self._combine_addresses(row),
axis=1,
)
datablock_compiled["bitrank"] = datablock["bitrank_def"] datablock_compiled["bitrank"] = datablock["bitrank_def"]
datablock_compiled["save"] = datablock["save"] datablock_compiled["save"] = datablock["save"]
datablock_compiled["comment"] = datablock.apply( datablock_compiled["comment"] = datablock.apply(
lambda row: f"{row['comment']}: {row['comment_def']}", axis=1 lambda row: self._combine_comments(
row["comment"], row["comment_def"]
),
axis=1,
) )
return DataBlock(datablock_compiled) return DataBlock(datablock_compiled)
@@ -454,23 +530,8 @@ class XSY:
) )
return pd.concat((edts, udts_finished)) return pd.concat((edts, udts_finished))
def _make_array_definitions(
self, datablock: DataBlock, definitions: Definitions
) -> Definitions:
"""Make definitions for the expansion of arrays of UDTs."""
everything_grouped = pd.concat((datablock, definitions)).groupby("type")
array_definitions_list = []
for name, group in everything_grouped:
try:
array = iec.Array(name)
if not array.is_elementary():
array_definitions_list.append(array.make_definition(""))
except iec.InvalidSpecError:
continue
return pd.concat(array_definitions_list)
class XSYReader(frame.Reader):
class XSYReader(Format):
"""Class for reading and writing Schneider Electric EcoStruxure Control """Class for reading and writing Schneider Electric EcoStruxure Control
Expert variable exchange (XSY) files. Can be re-used to read multiple XSY Expert variable exchange (XSY) files. Can be re-used to read multiple XSY
files using common stylesheets and UDT definitions. files using common stylesheets and UDT definitions.
@@ -483,7 +544,8 @@ class XSYReader(Format):
definitions_ss_path: os.path | None = None, definitions_ss_path: os.path | None = None,
even_addresses: bool = True, even_addresses: bool = True,
): ):
"""Initialize XSYReader object. """Initialize XSYReader object with default definition files and
stylesheets.
Parameters Parameters
---------- ----------
@@ -510,16 +572,19 @@ class XSYReader(Format):
"""Read the dataBlock and definitions of an XSY file into an XSY """Read the dataBlock and definitions of an XSY file into an XSY
object. object.
""" """
self.datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path) datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path)
self.definitions = Definitions.from_file( definitions = Definitions.from_file(
xsy_path, self.definitions_ss_path, self.definition_paths xsy_path, self.definitions_ss_path, self.definition_paths
) )
return XSY( xsy = XSY(
datablock=self.datablock, datablock=datablock,
definitions=self.definitions, definitions=definitions,
even_addresses=self.even_addresses, even_addresses=self.even_addresses,
name=os.path.basename(xsy_path), name=os.path.basename(xsy_path),
) )
if expand:
xsy.datablock = xsy.expand_datablock()
return xsy
def write(self): def write(self):
"""Write XSY object to file.""" """Write XSY object to file."""

View File

@@ -9,16 +9,17 @@ desc: Unit tests for the AuDaGe XSY module.
""" """
import os import os
# import time
import unittest import unittest
from importlib import resources from importlib import resources
import output
import assets
from AuDaGe.formats.controlexpert.xsy import * from AuDaGe.formats.controlexpert.xsy import *
try:
from tests import assets, output
except ImportError: # pragma: no cover
import assets # noqa: I001
import output
class TestXSY(unittest.TestCase): class TestXSY(unittest.TestCase):
"""This test case tests the XSYBlock, DataBlock, and Definitions classes in """This test case tests the XSYBlock, DataBlock, and Definitions classes in
@@ -76,7 +77,9 @@ class TestXSYReader(unittest.TestCase):
index=False, index=False,
) )
debug = False debug = False
for debug_step in XSY.DebugStep if debug else [XSY.DebugStep.NONE]: for debug_step in (
XSY.DebugStep if debug else [XSY.DebugStep.COMPLETE]
):
xsy.expand_datablock(debug_step=debug_step).to_csv( xsy.expand_datablock(debug_step=debug_step).to_csv(
self.out_path.joinpath( self.out_path.joinpath(
f"{file_name}_debug_{debug_step.name}.csv" f"{file_name}_debug_{debug_step.name}.csv"