release v0.10.1

pyproject.toml

@@ -61,7 +61,7 @@ pre-commit = "4.3.0"
 charset-normalizer = "3.4.6"
 python-discovery = "1.6.1"
 idna = "3.15"  # dependency of requests
-virtualenv = "21.14.9"
+virtualenv = "21.14.13"
 certifi = "2026.10.23"
 filelock = "4.2.1"
 
@@ -84,9 +84,9 @@ optional = true
 
 [tool.poetry.group.lint.dependencies]
 black = "26.10.1"
-astroid = "3.3.9"
+astroid = "4.3.4"
 isort = "5.13.2"
-pylint = "3.3.9"
+pylint = "4.1.3"
 mypy = "1.20.2"
 librt = "0.11.0"  # mypy dependency
 boto3-stubs = {extras = ["essential"], version = "1.26.72"}
@@ -157,8 +157,7 @@ ignore-patterns = ["^\\.#"]
 jobs = 1
 limit-inference-results = 100
 persistent = true
-py-version = "3.10"
-suggestion-mode = true
+py-version = "3.11"
 
 [tool.pylint.basic]
 argument-naming-style = "snake_case"

src/dve/core_engine/backends/base/reader.py

@@ -165,9 +165,10 @@ def read_to_entity_type(
                             entity=entity_name,
                             record=None,
                             failure_type="submission",
-                            error_location=entity_name,
                             error_code=config.error_code,
                             error_message=config.error_message,
+                            error_type="submission",
+                            category="Empty entity",
                         )
                     ],
                 )

src/dve/core_engine/backends/exceptions.py

@@ -79,7 +79,7 @@
         This will be joined to the location string with the joiner.
         """
 
-    def get_joiner(self):
+    def get_joiner(self) -> str:
         """The joiner between the preamble and the context string."""
         return "in"
 
@@ -194,7 +194,7 @@
         return f": {self.constraint}"
 
 
-class ReaderErrorMixin(BackendErrorMixin):
+class ReaderErrorMixin(BackendErrorMixin):  # pylint: disable=W0223
     """A mixin to be used by the reader errors."""
 
     def get_joiner(self) -> str:
@@ -233,41 +233,76 @@
         return f"Refdata loader does not support reading refdata from {self.file_extension} files"
 
 
-class EmptyFileError(ReaderErrorMixin, ValueError):
+class EmptyFileError(ReaderErrorMixin, CriticalMessageBearingError):
     """The read file was empty."""
 
     def get_message_preamble(self) -> str:
         return "File location provided is empty"
 
+    def __init__(self, *args: object, error_message: Optional[str] = None):
+
+        super().__init__(
+            *args,
+            message=FeedbackMessage(
+                entity="Whole File",
+                record=None,
+                failure_type="submission",
+                error_code="EMPTYFILE",
+                error_message=error_message or self.get_message_preamble(),
+            ),
+        )
+
 
-class MissingHeaderError(ReaderErrorMixin, ValueError):
+class MissingHeaderError(ReaderErrorMixin, CriticalMessageBearingError):
     """A header is expected, but was not parsed."""
 
     def get_message_preamble(self) -> str:
         return "No header was provided"
 
+    def __init__(self, *args: object, error_message: Optional[str] = None):
+        super().__init__(
+            *args,
+            message=FeedbackMessage(
+                entity="Whole File",
+                record=None,
+                failure_type="submission",
+                error_code="MISSINGHEADER",
+                error_message=error_message or self.get_message_preamble(),
+            ),
+        )
+
 
-class FieldCountMismatch(ReaderErrorMixin, ValueError):
+class FieldCountMismatch(ReaderErrorMixin, CriticalMessageBearingError):
     """An error raised when the number of expected fields does not match
     the number of parsed fields.
 
     """
 
     def __init__(self, *args: object, n_expected_fields: int, n_actual_fields: int) -> None:
-        super().__init__(*args)
         self.n_expected_fields = n_expected_fields
         """The number of fields that should have been in the file."""
         self.n_actual_fields = n_actual_fields
         """The number of fields that were actually in the file."""
 
+        super().__init__(
+            *args,
+            message=FeedbackMessage(
+                entity="Whole File",
+                record=None,
+                failure_type="submission",
+                error_code="FIELDCOUNTMISMATCH",
+                error_message=self.get_message_preamble(),
+            ),
+        )
+
     def get_message_preamble(self) -> str:
         return (
             f"Number of fields expected ({self.n_expected_fields}) does not "
             + f"match the number in the provided data ({self.n_actual_fields})"
         )
 
 
-class SchemaMismatch(ReaderErrorMixin, ValueError):
+class SchemaMismatch(ReaderErrorMixin, CriticalMessageBearingError):
     """An error raised when a file does not match a given schema (e.g. fields
     are missing).
 
@@ -290,7 +325,16 @@
         tuple of actual and expected field type, both as strings).
 
         """
-        super().__init__(*args)
+        super().__init__(
+            *args,
+            message=FeedbackMessage(
+                entity="Whole File",
+                record=None,
+                failure_type="submission",
+                error_code="SCHEMAMISMATCH",
+                error_message=self.get_message_preamble() + self.get_message_epilogue(),
+            ),
+        )
 
     def get_message_preamble(self) -> str:
         return "Schema mismatch"

src/dve/core_engine/backends/implementations/duckdb/contract.py

@@ -143,7 +143,7 @@ def apply_data_contract(
                 if relation_is_empty(relation):
                     self.logger.warning(f"+ Empty relation for {entity_name}")
                     empty_df = pl.DataFrame([], schema=polars_schema)  # type: ignore # pylint: disable=W0612
-                    relation = self._connection.sql("select * from empty_df")
+                    entities[entity_name] = self._connection.sql("select * from empty_df")
                     continue
 
                 self.logger.info(f"+ Applying contract to: {entity_name}")

src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -40,7 +40,7 @@
 @duckdb_check_entity_empty
 @duckdb_record_index
 @duckdb_write_parquet
-class DuckDBCSVReader(CSVFileReader):
+class DuckDBCSVReader(CSVFileReader):  # pylint: disable=W0223
     """A reader for CSV files including the ability to compare the passed model
     to the file header, if it exists.
 
@@ -99,7 +99,7 @@ def read_to_relation(  # pylint: disable=unused-argument
     ) -> DuckDBPyRelation:
         """Returns a relation object from the source csv"""
         if get_content_length(resource) == 0:
-            raise EmptyFileError(f"File at {resource} is empty.")
+            raise EmptyFileError(error_message=f"File at {resource} is empty.")
 
         if self.field_check:
             self.perform_field_check(resource, entity_name, schema, all_model_fields)
@@ -137,7 +137,7 @@ def read_to_relation(  # pylint: disable=unused-argument
 
 
 @polars_record_index
-class PolarsToDuckDBCSVReader(DuckDBCSVReader):
+class PolarsToDuckDBCSVReader(DuckDBCSVReader):  # pylint: disable=W0223
     """
     Utilises the polars lazy csv reader which is then converted into a DuckDBPyRelation object.
 
@@ -155,7 +155,7 @@ def read_to_relation(  # pylint: disable=unused-argument
     ) -> DuckDBPyRelation:
         """Returns a relation object from the source csv"""
         if get_content_length(resource) == 0:
-            raise EmptyFileError(f"File at {resource} is empty.")
+            raise EmptyFileError(error_message=f"File at {resource} is empty.")
 
         if self.field_check:
             self.perform_field_check(resource, entity_name, schema, all_model_fields)
@@ -209,7 +209,7 @@ def read_to_relation(  # pylint: disable=unused-argument
         return entity
 
 
-class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):
+class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):  # pylint: disable=W0223
     """A Reader for files with a `.csv` extension and where there are repeating "header" values
     within the file. Header in this case is not the column names at the top of a csv, rather a
     collection of unique records that would usually be structured in another entity. However, due

src/dve/core_engine/backends/implementations/duckdb/readers/json.py

@@ -22,7 +22,7 @@
 @duckdb_check_entity_empty
 @duckdb_record_index
 @duckdb_write_parquet
-class DuckDBJSONReader(BaseFileReader):
+class DuckDBJSONReader(BaseFileReader):  # pylint: disable=W0223
     """A reader for JSON files"""
 
     def __init__(

src/dve/core_engine/backends/implementations/duckdb/readers/xml.py

@@ -26,7 +26,7 @@
 @duckdb_check_entity_empty
 @polars_record_index
 @duckdb_write_parquet
-class DuckDBXMLStreamReader(XMLStreamReader):
+class DuckDBXMLStreamReader(XMLStreamReader):  # pylint: disable=W0223
     """A reader for XML files"""
 
     def __init__(self, *, connection: Optional[DuckDBPyConnection] = None, **kwargs):

src/dve/core_engine/backends/implementations/spark/contract.py

@@ -47,7 +47,7 @@
 @spark_record_index
 @spark_write_parquet
 @spark_read_parquet
-class SparkDataContract(BaseDataContract[DataFrame]):
+class SparkDataContract(BaseDataContract[DataFrame]):  # pylint: disable=W0223
     """An implementation of a data contract in Spark.
 
     This uses Spark's RDD functionality to apply the data contract.

src/dve/core_engine/backends/implementations/spark/readers/csv.py

@@ -24,7 +24,7 @@
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkCSVReader(CSVFileReader):
+class SparkCSVReader(CSVFileReader):  # pylint: disable=W0223
     """A Spark reader for CSV files."""
 
     # pylint: disable=R0902
@@ -80,7 +80,7 @@ def read_to_dataframe(
     ) -> DataFrame:
         """Read a CSV file directly to a Spark DataFrame."""
         if get_content_length(resource) == 0:
-            raise EmptyFileError(f"File at {resource} is empty.")
+            raise EmptyFileError(error_message=f"File at {resource} is empty.")
 
         if self.field_check:
             self.perform_field_check(resource, entity_name, schema, all_model_fields)

src/dve/core_engine/backends/implementations/spark/readers/json.py

@@ -22,7 +22,7 @@
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkJSONReader(BaseFileReader):
+class SparkJSONReader(BaseFileReader):  # pylint: disable=W0223
     """A Spark reader for JSON files."""
 
     def __init__(
@@ -60,7 +60,7 @@ def read_to_dataframe(
     ) -> DataFrame:
         """Read a JSON file directly to a Spark DataFrame."""
         if get_content_length(resource) == 0:
-            raise EmptyFileError(f"File at {resource} is empty.")
+            raise EmptyFileError(error_message=f"File at {resource} is empty.")
 
         spark_schema: StructType = get_type_from_annotation(schema)
         kwargs = {

src/dve/core_engine/backends/implementations/spark/readers/xml.py

@@ -33,7 +33,7 @@
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkXMLStreamReader(XMLStreamReader):
+class SparkXMLStreamReader(XMLStreamReader):  # pylint: disable=W0223
     """An XML stream reader that adds a method to read to a dataframe"""
 
     spark = None
@@ -61,7 +61,7 @@ def read_to_dataframe(
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkXMLReader(BasicXMLFileReader):  # pylint: disable=too-many-instance-attributes
+class SparkXMLReader(BasicXMLFileReader):  # pylint: disable=too-many-instance-attributes,W0223
     """A reader for XML files built atop Spark-XML."""
 
     def __init__(
@@ -130,7 +130,7 @@ def read_to_dataframe(
 
         """
         if get_content_length(resource) == 0:
-            raise EmptyFileError(f"File at {resource} is empty.")
+            raise EmptyFileError(error_message=f"File at {resource} is empty.")
 
         if self.xsd_location:
             msg = self._run_xmllint(file_uri=resource)

src/dve/core_engine/backends/implementations/spark/rules.py

@@ -61,7 +61,7 @@
 @spark_write_parquet
 @spark_read_parquet
 @spark_filter_contract_errors
-class SparkStepImplementations(BaseStepImplementations[DataFrame]):
+class SparkStepImplementations(BaseStepImplementations[DataFrame]):  # pylint: disable=W0223
     """An implementation of transformation steps in Apache Spark."""
 
     def __init__(self, spark_session: Optional[SparkSession] = None, **kwargs):