diff --git a/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt b/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt
index d1fd310..a75e5d9 100644
--- a/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt
+++ b/src/AuDaGe/formats/controlexpert/assets/flatten_datablock.xslt
@@ -19,12 +19,12 @@
-
+
+
- :
diff --git a/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt b/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt
index 093107a..5466d5f 100644
--- a/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt
+++ b/src/AuDaGe/formats/controlexpert/assets/flatten_ddtsource.xslt
@@ -24,9 +24,10 @@
-
+ :
+
@@ -35,7 +36,6 @@
- :
diff --git a/src/AuDaGe/formats/controlexpert/xsy.py b/src/AuDaGe/formats/controlexpert/xsy.py
index 51fee68..566f189 100644
--- a/src/AuDaGe/formats/controlexpert/xsy.py
+++ b/src/AuDaGe/formats/controlexpert/xsy.py
@@ -15,13 +15,14 @@ from collections.abc import Iterable
from dataclasses import dataclass
from enum import IntEnum, StrEnum, auto
from importlib import resources
+from itertools import pairwise
import pandas as pd
+from numpy import nan
-from AuDaGe import iec
-from AuDaGe.audage_frame import AuDaGeFrame
+from AuDaGe import frame, iec
+from AuDaGe.better_enum import BetterIntEnum
from AuDaGe.debug_tools import MeasureTime, TimeUnit
-from AuDaGe.format import Format
ASSETS_PATH = "AuDaGe.formats.controlexpert.assets"
@@ -31,23 +32,19 @@ MAX_RECURSION = 5
module_logger = logging.getLogger(__name__)
-class XSYBlock(AuDaGeFrame):
+class XSYBlock(frame.AuDaGeFrame):
"""Base class for DataBlock and Definitions that handles shared attributes.
May be instantiated directly from a dataframe or from an XSY file using the
from_file class method.
"""
- # _stylesheet_path: str # = None
- # _xsy_path: str # = None
- _name: str
-
@classmethod
def from_file(
cls,
xsy_path: os.PathLike,
stylesheet_path: os.PathLike,
name_ext: str = "",
- ) -> XSYBlock: # noqa: F821
+ ) -> XSYBlock:
"""Load an XSY file into a dataframe using the specified stylesheet."""
cls.stylesheet_path = stylesheet_path
cls.xsy_path = xsy_path
@@ -55,7 +52,6 @@ class XSYBlock(AuDaGeFrame):
resources.as_file(cls.xsy_path) as xsy_file,
resources.as_file(cls.stylesheet_path) as stylesheet_file,
):
- # frame = AuDaGeFrame(pd.read_xml(xsy_file, stylesheet=stylesheet_file))
return cls(
pd.read_xml(xsy_file, stylesheet=stylesheet_file),
".".join(
@@ -63,6 +59,26 @@ class XSYBlock(AuDaGeFrame):
),
)
+ def _make_array_definitions(
+ self, frame: frame.AuDaGeFrame | None = None
+ ) -> frame.AuDaGeFrame | None:
+ """Make definitions for the expansion of arrays of UDTs."""
+ frame = self if frame is None else frame
+ frame_grouped = frame.groupby("type")
+ array_definitions_list = []
+ for name, group in frame_grouped:
+ try:
+ array = iec.Array(name)
+ if not array.is_elementary():
+ array_definitions_list.append(array.make_definition(""))
+ except iec.InvalidSpecError:
+ continue
+ try:
+ return pd.concat(array_definitions_list)
+ # Return `None` if there is nothing to concatenate.
+ except ValueError:
+ return None
+
class DataBlock(XSYBlock):
"""Read and parse the dataBlock node of an XSY file."""
@@ -74,11 +90,11 @@ class DataBlock(XSYBlock):
@classmethod
def from_file(
cls, xsy_path: os.PathLike, stylesheet_path: os.PathLike | None = None
- ) -> DataBlock: # noqa: F821
+ ) -> DataBlock:
"""Read the dataBlock of an XSY file into a DataFrame."""
if stylesheet_path is None:
stylesheet_path = cls.DEFAULT_SS_PATH
- return cls(XSYBlock.from_file(xsy_path, stylesheet_path, "DataBlock"))
+ return cls(super().from_file(xsy_path, stylesheet_path, "DataBlock"))
class Definitions(XSYBlock):
@@ -91,6 +107,14 @@ class Definitions(XSYBlock):
"builtins.xsy"
)
+ class BoolParents(BetterIntEnum):
+ """Enum of UDTs that pack booleans into data blocks other than bits or
+ words. Each member name is the name of the UDT and the value is the
+ number of bits that each boolean is packed into.
+ """
+
+ PARA_PI_B = 8
+
@classmethod
def from_file(
cls,
@@ -98,7 +122,7 @@ class Definitions(XSYBlock):
stylesheet_path: os.path | None = None,
definition_paths: Iterable[os.path] | None = None,
include_builtins: bool = True,
- ) -> Definitions: # noqa: F821
+ ) -> Definitions:
"""Read the DDTSource nodes of an XSY file into a DataFrame."""
cls.stylesheet_path = (
cls.DEFAULT_SS_PATH if stylesheet_path is None else stylesheet_path
@@ -122,38 +146,71 @@ class Definitions(XSYBlock):
_name = ".".join(
(os.path.splitext(os.path.basename(xsy_path))[0], "Definitions")
)
- with MeasureTime(
- TimeUnit.MS,
- True,
+ definitions = XSYBlock(pd.concat(cls.definitions_list))
+ definitions_extended = pd.concat(
+ (definitions, definitions._make_array_definitions())
+ )
+ time_message = (
"{_name} offset calculation completed in {time} {unit}.".format(
_name=_name, time="{time:.3f}", unit="{unit}"
- ),
- ):
- offsets = cls._calculate_offsets(
- pd.concat(cls.definitions_list),
- _name=_name,
)
+ )
+ with MeasureTime(TimeUnit.MS, True, time_message):
+ offsets = cls._calculate_offsets(definitions_extended, _name=_name)
return cls(offsets)
- def _get_length(self, row, udt_lengths: dict[str, int]) -> int:
- # Find packed-bit words by checking whether...
+ @classmethod
+ def _get_length(
+ cls,
+ row,
+ udt_lengths: dict[str, int],
+ bool_parents: IntEnum = BoolParents,
+ ) -> int:
+ # Special case to fix the BOOLs that are packed into data units other
+ # than bits or words.
if (
- # ...the first member is a 16-bit binary or integer type...
- row["type"] in ("WORD", "INT", "UINT")
- # ...and that the second member is a boolean...
- and row["next_type"] == "BOOL"
- # ...and that the second member has a bitrank.
- and row["next_bitrank"].notna()
+ bool_parents.contains(row["parent"].upper())
+ and row["type"] == "BOOL"
):
- # We don't increase the offset for packed-bit words because the
- # following booleans overlap with them.
+ return bool_parents[row["parent"].upper()].value
+ # Bit- and byte- length variables that are packed into words have
+ # functionally zero length for our purposes.
+ if row["type"] in (
+ "BOOL",
+ "EBOOL",
+ "BYTE",
+ "SINT",
+ "USINT",
+ "CHAR",
+ ) and pd.notna(row["bitrank"]):
return 0
+ length = iec.get_data_length(
+ row["type"],
+ udt_lengths=udt_lengths,
+ unknown_length=nan,
+ )
+ if pd.isna(length):
+ return length
else:
- return iec.get_data_length(
- row["type"],
- udt_lengths=udt_lengths,
- unknown_length=pd.NA,
- )
+ return max(length, 16)
+
+ @classmethod
+ def _calculate_padding(
+ cls,
+ address: int | nan,
+ length: int | nan,
+ max_alignment: int = iec.DataUnit.DWORD,
+ ) -> int | nan:
+ # Return early if missing information.
+ if pd.isna(address) or pd.isna(length) or length <= 0:
+ return address
+ alignment = min(length, max_alignment)
+ rem = address % alignment
+ # No change is necessary if address is divisible by length.
+ if rem <= 0:
+ return address
+ # Increase address to make it divisible by length.
+ return address + alignment - rem
@classmethod
def _calculate_offsets(
@@ -163,7 +220,7 @@ class Definitions(XSYBlock):
max_recursion: int = MAX_RECURSION,
_current_recursion: int = 0,
_name: str = "",
- ) -> Definitions: # noqa: F821
+ ) -> Definitions:
"""Recursively calculate the memory offset for all UDT members in the
definitions.
"""
@@ -183,30 +240,11 @@ class Definitions(XSYBlock):
udt_lengths = {} if udt_lengths is None else udt_lengths
# We don't care about DFBs so set their length at zero.
udt_lengths["DFB"] = 0
- definitions_joined = definitions.shift((0, -1), axis=0)
- definitions_joined.to_csv("definitions_joined.csv")
# Create a column with the length of each member.
- definitions["length"] = definitions_joined.apply(
- lambda row: (
- 0
- if (
- row["type_0"] in ("WORD", "INT", "UINT")
- and row["type_-1"] == "BOOL"
- and pd.notna(row["bitrank_-1"])
- )
- else iec.get_data_length(
- row["type_0"],
- udt_lengths=udt_lengths,
- unknown_length=pd.NA,
- )
- ),
+ definitions["length"] = definitions.apply(
+ lambda row: cls._get_length(row, udt_lengths),
axis=1,
- ).shift(1, axis=0, fill_value=0)
- definitions.to_csv("definitions.csv")
- # Calculate how many bits need to be skipped before each member so
- # that addressing rules are followed (e.g. word-length types on
- # integer addresses, multi-word types on even addresses).
- # definitions["padding"] = definitions.apply()
+ )
# Set a flag high for checking whether all offsets have been found.
# If this isn't set False by anything later then we're done.
all_offsets_found = True
@@ -214,16 +252,24 @@ class Definitions(XSYBlock):
# definition. This is the offset of the subsequent member.
definitions_calculated_list = []
for name, group in definitions.groupby("parent"):
- group["offset"] = group["length"].cumsum(skipna=False)
+ group_dict = group.to_dict("records")
+ group_dict[0]["address"] = 0
+ group_dict_calc = [group_dict[0]]
+ for prev_row, row in pairwise(group_dict):
+ row["address"] = cls._calculate_padding(
+ prev_row["address"] + prev_row["length"], row["length"]
+ )
+ group_dict_calc.append(row)
+ group_calc = frame.AuDaGeFrame(group_dict_calc)
# Check if all offsets for this UDT has been found. If so, we
# know its length so we save it to the `udt_lengths` dictionary
# for future reference.
- if group["offset"].notna().all():
- udt_lengths[name] = group["offset"].tail(1).values[0]
+ if group_calc["address"].notna().all():
+ udt_lengths[name] = group_calc["address"].tail(1).values[0]
else:
# This UDT's offsets aren't finished so set the flag False.
all_offsets_found = False
- definitions_calculated_list.append(group)
+ definitions_calculated_list.append(group_calc)
# Re-combine all the UDT definitions.
definitions_calculated = pd.concat(definitions_calculated_list)
if all_offsets_found:
@@ -269,7 +315,7 @@ class XSY:
GROUP = auto() # 2
JOIN = auto() # 3
COMPILE = auto() # 4
- NONE = auto() # 5
+ COMPLETE = auto() # 5
class IgnoredDataTypes(StrEnum):
DFB = auto()
@@ -279,7 +325,7 @@ class XSY:
datablock: DataBlock | None = None,
definitions: Definitions | None = None,
ignored_data_types: StrEnum = IgnoredDataTypes,
- debug_step: DebugStep | int = DebugStep.NONE,
+ debug_step: DebugStep | int = DebugStep.COMPLETE,
max_recursion: int = MAX_RECURSION,
) -> DataBlock:
"""Recursively expand the UDTs in `datablock` based on the UDT
@@ -321,11 +367,8 @@ class XSY:
unit="{unit}",
),
):
- array_definitions = self._make_array_definitions(
- datablock, definitions
- )
definitions_extended = pd.concat(
- (definitions, array_definitions)
+ (definitions, datablock._make_array_definitions())
)
if debug_step == self.DebugStep.MAKE_ARRAY_DEFS:
return definitions_extended
@@ -338,6 +381,33 @@ class XSY:
)
return datablock_expanded
+ def _combine_addresses(self, row: pd.Series) -> iec.Address:
+ if pd.isna(row["address"]):
+ return nan
+ address_object = iec.Address.from_string(row["address"])
+ if pd.notna(row["address_def"]):
+ offset = row["address_def"] // 16
+ else:
+ offset = 0
+ if pd.notna(row["bitrank_def"]):
+ bit = float(row["bitrank_def"])
+ if address_object.bit is None:
+ address_object.bit = bit
+ else:
+ address_object.bit += bit
+ offset -= 1
+ address_object.word += offset
+ return str(address_object)
+
+ def _combine_comments(self, *comments: str) -> str:
+ comments_cleaned = []
+ for comment in comments:
+ if pd.isna(comment) or comment == "":
+ continue
+ comment_cleaned = comment.lstrip(": ")
+ comments_cleaned.append(comment_cleaned)
+ return ": ".join(comments_cleaned)
+
def _compile_datablock(self, datablock: DataBlock) -> DataBlock:
"""Compile a cross-joined and filtered DataBlock by combining the
columns from the datablock with the columns from the definitions."""
@@ -347,11 +417,17 @@ class XSY:
)
datablock_compiled["parent"] = ""
datablock_compiled["type"] = datablock["type_def"]
- datablock_compiled["address"] = datablock["address"]
+ datablock_compiled["address"] = datablock.apply(
+ lambda row: self._combine_addresses(row),
+ axis=1,
+ )
datablock_compiled["bitrank"] = datablock["bitrank_def"]
datablock_compiled["save"] = datablock["save"]
datablock_compiled["comment"] = datablock.apply(
- lambda row: f"{row['comment']}: {row['comment_def']}", axis=1
+ lambda row: self._combine_comments(
+ row["comment"], row["comment_def"]
+ ),
+ axis=1,
)
return DataBlock(datablock_compiled)
@@ -454,23 +530,8 @@ class XSY:
)
return pd.concat((edts, udts_finished))
- def _make_array_definitions(
- self, datablock: DataBlock, definitions: Definitions
- ) -> Definitions:
- """Make definitions for the expansion of arrays of UDTs."""
- everything_grouped = pd.concat((datablock, definitions)).groupby("type")
- array_definitions_list = []
- for name, group in everything_grouped:
- try:
- array = iec.Array(name)
- if not array.is_elementary():
- array_definitions_list.append(array.make_definition(""))
- except iec.InvalidSpecError:
- continue
- return pd.concat(array_definitions_list)
-
-class XSYReader(Format):
+class XSYReader(frame.Reader):
"""Class for reading and writing Schneider Electric EcoStruxure Control
Expert variable exchange (XSY) files. Can be re-used to read multiple XSY
files using common stylesheets and UDT definitions.
@@ -483,7 +544,8 @@ class XSYReader(Format):
definitions_ss_path: os.path | None = None,
even_addresses: bool = True,
):
- """Initialize XSYReader object.
+ """Initialize XSYReader object with default definition files and
+ stylesheets.
Parameters
----------
@@ -510,16 +572,19 @@ class XSYReader(Format):
"""Read the dataBlock and definitions of an XSY file into an XSY
object.
"""
- self.datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path)
- self.definitions = Definitions.from_file(
+ datablock = DataBlock.from_file(xsy_path, self.datablock_ss_path)
+ definitions = Definitions.from_file(
xsy_path, self.definitions_ss_path, self.definition_paths
)
- return XSY(
- datablock=self.datablock,
- definitions=self.definitions,
+ xsy = XSY(
+ datablock=datablock,
+ definitions=definitions,
even_addresses=self.even_addresses,
name=os.path.basename(xsy_path),
)
+ if expand:
+ xsy.datablock = xsy.expand_datablock()
+ return xsy
def write(self):
"""Write XSY object to file."""
diff --git a/tests/test_xsy.py b/tests/test_xsy.py
index 5312a9b..b2d90d6 100644
--- a/tests/test_xsy.py
+++ b/tests/test_xsy.py
@@ -9,16 +9,17 @@ desc: Unit tests for the AuDaGe XSY module.
"""
import os
-
-# import time
import unittest
from importlib import resources
-import output
-
-import assets
from AuDaGe.formats.controlexpert.xsy import *
+try:
+ from tests import assets, output
+except ImportError: # pragma: no cover
+ import assets # noqa: I001
+ import output
+
class TestXSY(unittest.TestCase):
"""This test case tests the XSYBlock, DataBlock, and Definitions classes in
@@ -76,7 +77,9 @@ class TestXSYReader(unittest.TestCase):
index=False,
)
debug = False
- for debug_step in XSY.DebugStep if debug else [XSY.DebugStep.NONE]:
+ for debug_step in (
+ XSY.DebugStep if debug else [XSY.DebugStep.COMPLETE]
+ ):
xsy.expand_datablock(debug_step=debug_step).to_csv(
self.out_path.joinpath(
f"{file_name}_debug_{debug_step.name}.csv"