Finish address offset calculation for XSY module.

This commit is contained in:
Joshua Lucas
2026-10-14 21:23:21 -05:00
parent 0aaa9492b9
commit 392726a906
4 changed files with 171 additions and 103 deletions

View File

@@ -19,12 +19,12 @@
<xsl:template match="variables">
<xsl:copy>
<name><xsl:value-of select="@name"/></name>
<parent/>
<type><xsl:value-of select="@typeName"/></type>
<address><xsl:value-of select="@topologicalAddress"/></address>
<comment><xsl:value-of select="comment"/></comment>
<parent/>
<bitrank/>
<save><xsl:value-of select="attribute/@Save"/></save>
<comment><xsl:value-of select="../../comment"/>: <xsl:value-of select="comment"/></comment>
</xsl:copy>
</xsl:template>

View File

@@ -24,9 +24,10 @@
<xsl:template match="structure/variables">
<xsl:copy>
<name><xsl:value-of select="@name"/></name>
<parent><xsl:value-of select="translate(../../@DDTName, $lowercase, $uppercase)"/></parent>
<type><xsl:value-of select="translate(@typeName, $lowercase, $uppercase)"/></type>
<address/>
<comment><xsl:value-of select="../../comment"/>: <xsl:value-of select="comment"/></comment>
<parent><xsl:value-of select="translate(../../@DDTName, $lowercase, $uppercase)"/></parent>
<bitrank>
<xsl:for-each select="attribute">
<xsl:if test="@name='ExtractBit'">
@@ -35,7 +36,6 @@
</xsl:for-each>
</bitrank>
<save/>
<comment><xsl:value-of select="../../comment"/>: <xsl:value-of select="comment"/></comment>
</xsl:copy>
</xsl:template>

View File

@@ -15,13 +15,14 @@ from collections.abc import Iterable
from dataclasses import dataclass
from enum import IntEnum, StrEnum, auto
from importlib import resources
from itertools import pairwise
import pandas as pd
from numpy import nan
from AuDaGe import iec
from AuDaGe.audage_frame import AuDaGeFrame
from AuDaGe import frame, iec
from AuDaGe.better_enum import BetterIntEnum
from AuDaGe.debug_tools import MeasureTime, TimeUnit
from AuDaGe.format import Format
ASSETS_PATH = "AuDaGe.formats.controlexpert.assets"
@@ -31,23 +32,19 @@ MAX_RECURSION = 5
module_logger = logging.getLogger(__name__)
class XSYBlock(AuDaGeFrame):
class XSYBlock(frame.AuDaGeFrame):
"""Base class for DataBlock and Definitions that handles shared attributes.
May be instantiated directly from a dataframe or from an XSY file using the
from_file class method.
"""
# _stylesheet_path: str # = None
# _xsy_path: str # = None
_name: str
@classmethod
def from_file(
cls,
xsy_path: os.PathLike,
stylesheet_path: os.PathLike,
name_ext: str = "",
) -> XSYBlock: # noqa: F821
) -> XSYBlock:
"""Load an XSY file into a dataframe using the specified stylesheet."""
cls.stylesheet_path = stylesheet_path
cls.xsy_path = xsy_path
@@ -55,7 +52,6 @@ class XSYBlock(AuDaGeFrame):
resources.as_file(cls.xsy_path) as xsy_file,
resources.as_file(cls.stylesheet_path) as stylesheet_file,
):
# frame = AuDaGeFrame(pd.read_xml(xsy_file, stylesheet=stylesheet_file))
return cls(
pd.read_xml(xsy_file, stylesheet=stylesheet_file),
".".join(
@@ -63,6 +59,26 @@ class XSYBlock(AuDaGeFrame):
),
)
def _make_array_definitions(
self, frame: frame.AuDaGeFrame | None = None
) -> frame.AuDaGeFrame | None:
"""Make definitions for the expansion of arrays of UDTs."""
frame = self if frame is None else frame
frame_grouped = frame.groupby("type")
array_definitions_list = []
for name, group in frame_grouped:
try:
array = iec.Array(name)
if not array.is_elementary():
array_definitions_list.append(array.make_definition(""))
except iec.InvalidSpecError:
continue
try:
return pd.concat(array_definitions_list)
# Return `None` if there is nothing to concatenate.
except ValueError:
return None
class DataBlock(XSYBlock):
"""Read and parse the dataBlock node of an XSY file."""
@@ -74,11 +90,11 @@ class DataBlock(XSYBlock):
@classmethod
def from_file(
cls, xsy_path: os.PathLike, stylesheet_path: os.PathLike | None = None
) -> DataBlock: # noqa: F821
) -> DataBlock:
"""Read the dataBlock of an XSY file into a DataFrame."""
if stylesheet_path is None:
stylesheet_path = cls.DEFAULT_SS_PATH
return cls(XSYBlock.from_file(xsy_path, stylesheet_path, "DataBlock"))
return cls(super().from_file(xsy_path, stylesheet_path, "DataBlock"))
class Definitions(XSYBlock):
@@ -91,6 +107,14 @@ class Definitions(XSYBlock):
"builtins.xsy"
)
class BoolParents(BetterIntEnum):
"""Enum of UDTs that pack booleans into data blocks other than bits or
words. Each member name is the name of the UDT and the value is the
number of bits that each boolean is packed into.
"""
PARA_PI_B = 8
@classmethod
def from_file(
cls,
@@ -98,7 +122,7 @@ class Definitions(XSYBlock):
stylesheet_path: os.path | None = None,
definition_paths: Iterable[os.path] | None = None,
include_builtins: bool = True,
) -> Definitions: # noqa: F821
) -> Definitions:
"""Read the DDTSource nodes of an XSY file into a DataFrame."""
cls.stylesheet_path = (
cls.DEFAULT_SS_PATH if stylesheet_path is None else stylesheet_path
@@ -122,38 +146,71 @@ class Definitions(XSYBlock):
_name = ".".join(
(os.path.splitext(os.path.basename(xsy_path))[0], "Definitions")
)
with MeasureTime(
TimeUnit.MS,
True,
definitions = XSYBlock(pd.concat(cls.definitions_list))
definitions_extended = pd.concat(
(definitions, definitions._make_array_definitions())
)
time_message = (
"{_name} offset calculation completed in {time} {unit}.".format(
_name=_name, time="{time:.3f}", unit="{unit}"
),
):
offsets = cls._calculate_offsets(
pd.concat(cls.definitions_list),
_name=_name,
)
)
with MeasureTime(TimeUnit.MS, True, time_message):
offsets = cls._calculate_offsets(definitions_extended, _name=_name)
return cls(offsets)
def _get_length(self, row, udt_lengths: dict[str, int]) -> int:
# Find packed-bit words by checking whether...
@classmethod
def _get_length(
cls,
row,
udt_lengths: dict[str, int],
bool_parents: IntEnum = BoolParents,
) -> int:
# Special case to fix the BOOLs that are packed into data units other
# than bits or words.
if (
# ...the first member is a 16-bit binary or integer type...
row["type"] in ("WORD", "INT", "UINT")
# ...and that the second member is a boolean...
and row["next_type"] == "BOOL"
# ...and that the second member has a bitrank.
and row["next_bitrank"].notna()
bool_parents.contains(row["parent"].upper())
and row["type"] == "BOOL"
):
# We don't increase the offset for packed-bit words because the
# following booleans overlap with them.
return bool_parents[row["parent"].upper()].value
# Bit- and byte- length variables that are packed into words have
# functionally zero length for our purposes.
if row["type"] in (
"BOOL",
"EBOOL",
"BYTE",
"SINT",
"USINT",
"CHAR",
) and pd.notna(row["bitrank"]):
return 0
length = iec.get_data_length(
row["type"],
udt_lengths=udt_lengths,
unknown_length=nan,
)
if pd.isna(length):
return length
else:
return iec.get_data_length(
row["type"],
udt_lengths=udt_lengths,
unknown_length=pd.NA,
)
return max(length, 16)
@classmethod
def _calculate_padding(
cls,
address: int | nan,
length: int | nan,
max_alignment: int = iec.DataUnit.DWORD,
) -> int | nan:
# Return early if missing information.
if pd.isna(address) or pd.isna(length) or length <= 0:
return address
alignment = min(length, max_alignment)
rem = address % alignment
# No change is necessary if address is divisible by length.
if rem <= 0:
return address
# Increase address to make it divisible by length.
return address + alignment - rem
@classmethod
def _calculate_offsets(
@@ -163,7 +220,7 @@ class Definitions(XSYBlock):
max_recursion: int = MAX_RECURSION,
_current_recursion: int = 0,
_name: str = "",
) -> Definitions: # noqa: F821
) -> Definitions:
"""Recursively calculate the memory offset for all UDT members in the
definitions.
"""
@@ -183,30 +240,11 @@ class Definitions(XSYBlock):
udt_lengths = {} if udt_lengths is None else udt_lengths
# We don't care about DFBs so set their length at zero.
udt_lengths["DFB"] = 0
definitions_joined = definitions.shift((0, -1), axis=0)
definitions_joined.to_csv("definitions_joined.csv")
# Create a column with the length of each member.
definitions["length"] = definitions_joined.apply(
lambda row: (
0
if (
row["type_0"] in ("WORD", "INT", "UINT")
and row["type_-1"] == "BOOL"
and pd.notna(row["bitrank_-1"])
)
else iec.get_data_length(
row["type_0"],
udt_lengths=udt_lengths,
unknown_length=pd.NA,
)
),
definitions["length"] = definitions.apply(
lambda row: cls._get_length(row, udt_lengths),
axis=1,
).shift(1, axis=0, fill_value=0)
definitions.to_csv("definitions.csv")
# Calculate how many bits need to be skipped before each member so
# that addressing rules are followed (e.g. word-length types on
# integer addresses, multi-word types on even addresses).
# definitions["padding"] = definitions.apply()
)
# Set a flag high for checking whether all offsets have been found.
# If this isn't set False by anything later then we're done.
all_offsets_found = True
@@ -214,16 +252,24 @@ class Definitions(XSYBlock):
# definition. This is the offset of the subsequent member.
definitions_calculated_list = []
for name, group in definitions.groupby("parent"):
group["offset"] = group["length"].cumsum(skipna=False)
group_dict = group.to_dict("records")
group_dict[0]["address"] = 0
group_dict_calc = [group_dict[0]]
for prev_row, row in pairwise(group_dict):
row["address"] = cls._calculate_padding(
prev_row["address"] + prev_row["length"], row["length"]
)
group_dict_calc.append(row)
group_calc = frame.AuDaGeFrame(group_dict_calc)
# Check if all offsets for this UDT has been found. If so, we
# know its length so we save it to the `udt_lengths` dictionary
# for future reference.
if group["offset"].notna().all():
udt_lengths[name] = group["offset"].tail(1).values[0]
if group_calc["address"].notna().all():
udt_lengths[name] = group_calc["address"].tail(1).values[0]
else:
# This UDT's offsets aren't finished so set the flag False.
all_offsets_found = False
definitions_calculated_list.append(group)
definitions_calculated_list.append(group_calc)
# Re-combine all the UDT definitions.
definitions_calculated = pd.concat(definitions_calculated_list)
if all_offsets_found:
@@ -269,7 +315,7 @@ class XSY:
GROUP = auto() # 2
JOIN = auto() # 3
COMPILE = auto() # 4
NONE = auto() # 5
COMPLETE = auto() # 5
class IgnoredDataTypes(StrEnum):
DFB = auto()
@@ -279,7 +325,7 @@ class XSY:
datablock: DataBlock | None = None,
definitions: Definitions | None = None,
ignored_data_types: StrEnum = IgnoredDataTypes,
debug_step: DebugStep | int = DebugStep.NONE,
debug_step: DebugStep | int = DebugStep.COMPLETE,
max_recursion: int = MAX_RECURSION,
) -> DataBlock:
"""Recursively expand the UDTs in `datablock` based on the UDT
@@ -321,11 +367,8 @@ class XSY:
unit="{unit}",
),
):
array_definitions = self._make_array_definitions(
datablock, definitions
)
definitions_extended = pd.concat(
(definitions, array_definitions)
(definitions, datablock._make_array_definitions())
)
if debug_step == self.DebugStep.MAKE_ARRAY_DEFS:
return definitions_extended
@@ -338,6 +381,33 @@ class XSY:
)
return datablock_expanded
def _combine_addresses(self, row: pd.Series) -> iec.Address:
if pd.isna(row["address"]):
return nan
address_object = iec.Address.from_string(row["address"])
if pd.notna(row["address_def"]):
offset = row["address_def"] // 16
else:
offset = 0
if pd.notna(row["bitrank_def"]):
bit = float(row["bitrank_def"])
if address_object.bit is None:
address_object.bit = bit
else:
address_object.bit += bit
offset -= 1
address_object.word += offset
return str(address_object)
def _combine_comments(self, *comments: str) -> str:
comments_cleaned = []
for comment in comments:
if pd.isna(comment) or comment == "":
continue
comment_cleaned = comment.lstrip(": ")
comments_cleaned.append(comment_cleaned)
return ": ".join(comments_cleaned)
def _compile_datablock(self, datablock: DataBlock) -> DataBlock:
"""Compile a cross-joined and filtered DataBlock by combining the
columns from the datablock with the columns from the definitions."""
@@ -347,11 +417,17 @@ class XSY:
)
datablock_compiled["parent"] = ""
datablock_compiled["type"] = datablock["type_def"]
datablock_compiled["address"] = datablock["address"]
datablock_compiled["address"] = datablock.apply(
lambda row: self._combine_addresses(row),
axis=1,
)
datablock_compiled["bitrank"] = datablock["bitrank_def"]
datablock_compiled["save"] = datablock["save"]
datablock_compiled["comment"] = datablock.apply(
lambda row: f"{row['comment']}: {row['comment_def']}", axis=1
lambda row: self._combine_comments(
row["comment"], row["comment_def"]
),
axis=1,
)
return DataBlock(datablock_compiled)
@@ -454,23 +530,8 @@ class XSY:
)
return pd.concat((edts, udts_finished))
def _make_array_definitions(
self, datablock: DataBlock, definitions: Definitions
) -> Definitions:
"""Make definitions for the expansion of arrays of UDTs."""
everything_grouped = pd.concat((datablock, definitions)).groupby("type")
array_definitions_list = []
for name, group in everything_grouped:
try:
array = iec.Array(name)
if not array.is_elementary():
array_definitions_list.append(array.make_definition(""))
except iec.InvalidSpecError:
continue
return pd.concat(array_definitions_list)
class XSYReader(Format):
class XSYReader(frame.Reader):
"""Class for reading and writing Schneider Electric EcoStruxure Control
Expert variable exchange (XSY) files. Can be re-used to read multiple XSY
files using common stylesheets and UDT definitions.
@@ -483,7 +544,8 @@ class XSYReader(Format):
definitions_ss_path: os.path | None = None,
even_addresses: bool = True,
):
"""Initialize XSYReader object.
"""Initialize XSYReader object with default definition files and
stylesheets.
Parameters
----------
@@ -510,16 +572,19 @@ class XSYReader(Format):
"""Read the dataBlock and definitions of an XSY file into an XSY
object.
"""
self.datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path)
self.definitions = Definitions.from_file(
datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path)
definitions = Definitions.from_file(
xsy_path, self.definitions_ss_path, self.definition_paths
)
return XSY(
datablock=self.datablock,
definitions=self.definitions,
xsy = XSY(
datablock=datablock,
definitions=definitions,
even_addresses=self.even_addresses,
name=os.path.basename(xsy_path),
)
if expand:
xsy.datablock = xsy.expand_datablock()
return xsy
def write(self):
"""Write XSY object to file."""

View File

@@ -9,16 +9,17 @@ desc: Unit tests for the AuDaGe XSY module.
"""
import os
# import time
import unittest
from importlib import resources
import output
import assets
from AuDaGe.formats.controlexpert.xsy import *
try:
from tests import assets, output
except ImportError: # pragma: no cover
import assets # noqa: I001
import output
class TestXSY(unittest.TestCase):
"""This test case tests the XSYBlock, DataBlock, and Definitions classes in
@@ -76,7 +77,9 @@ class TestXSYReader(unittest.TestCase):
index=False,
)
debug = False
for debug_step in XSY.DebugStep if debug else [XSY.DebugStep.NONE]:
for debug_step in (
XSY.DebugStep if debug else [XSY.DebugStep.COMPLETE]
):
xsy.expand_datablock(debug_step=debug_step).to_csv(
self.out_path.joinpath(
f"{file_name}_debug_{debug_step.name}.csv"