diff --git a/poetry.lock b/poetry.lock index 6614bbf2..861553ba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -29,14 +29,14 @@ test = ["coverage", "mypy", "pexpect", "ruff", "wheel"] [[package]] name = "astroid" -version = "3.3.9" +version = "4.3.4" description = "An abstract syntax tree for Python with inference support." optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["dev", "lint"] files = [ - {file = "astroid-3.3.9-py3-none-any.whl", hash = "sha256:d05bfd0acba96a7bd43e222828b7d9bc1e138aaeb0649707908d3702a9831248"}, - {file = "astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550"}, + {file = "astroid-4.3.4-py3-none-any.whl", hash = "sha256:2bcd0d02648a443a4b818c952c3550091989daefac3c12d3b83b2289482e0818"}, + {file = "astroid-4.3.4.tar.gz", hash = "sha256:d515a105722b72098bbe82d430d65e635f742b6cbac3bdfaf8b7c188b87c5e39"}, ] [[package]] @@ -2684,24 +2684,24 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pylint" -version = "3.3.9" +version = "4.1.3" description = "python code static checker" optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["dev", "lint"] files = [ - {file = "pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7"}, - {file = "pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a"}, + {file = "pylint-4.1.3-py3-none-any.whl", hash = "sha256:a85357cae24f33ad8d86c8f3daaa92c600ae4012b54a57299cee76000e9364cf"}, + {file = "pylint-4.1.3.tar.gz", hash = "sha256:9928603068edfa0d1a3c167f174b099d4b97c3db75d32d0fcdd029770b4713a9"}, ] [package.dependencies] -astroid = ">=3.3.8,<=3.4.0.dev0" +astroid = ">=4.3.4,<=4.4" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.6", markers = "python_version == \"3.11\""}, {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, ] -isort = ">=4.2.5,<5.13 || >5.13,<7" +isort = ">=5,<5.13 || >5.13,<10" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2" tomlkit = ">=0.10.1" @@ -3324,14 +3324,14 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "virtualenv" -version = "21.14.9" +version = "21.14.13" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "virtualenv-21.14.9-py3-none-any.whl", hash = "sha256:ac51744da66ae56ff814b087d65096c94de698ec520adc624b108b679655809b"}, - {file = "virtualenv-21.14.9.tar.gz", hash = "sha256:4d1bee354cd81c820da3bee56e38be3426490f8d1fc54e10ea5b2072ebad8ef8"}, + {file = "virtualenv-21.14.13-py3-none-any.whl", hash = "sha256:5fbb04138a4861580b78ba9110d529a26bc48fcc93c8ec36bc0202eeed96a788"}, + {file = "virtualenv-21.14.13.tar.gz", hash = "sha256:de5307ba4b9a0001d9c8ea02832d38f63ca432880f416c1bface61c4323fd905"}, ] [package.dependencies] @@ -3555,7 +3555,6 @@ files = [ {file = "zensical-0.0.63-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:069af2ee0254eaa7099faad908f0ae0017c151ef0160b21a039d08efaa61b38a"}, {file = "zensical-0.0.63-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:6ec02283d946569f68fdd1ab4a1dd2de560c0ad540874d5ecd4a4f1e96bdbe1d"}, {file = "zensical-0.0.63-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af2711ff1397338c730cc9232f687066da7181751257f317877501b1b77a44ea"}, - {file = "zensical-0.0.63-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f1afb0e2819a588d956c1684c90e23fc0c7efc89e1ec141403a9fb41b75b1993"}, {file = "zensical-0.0.63-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925b3dd8bb6812b780f5e00828c42bd1eebd675b1eb6b4a58135926bb716bdde"}, {file = "zensical-0.0.63-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6365bce465a7f6755533eaa80aa20c8a48d1b39fa7b12cd394eece3921e2c530"}, {file = "zensical-0.0.63-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:63706419125f4c46461b322c8aaed1bbc3b917dd389ecf29601d713fc962bfb6"}, @@ -3597,4 +3596,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "889e54ad662e9460ae037f9a541c4f4f3993ee7062e81f58d312622fd615b4a7" +content-hash = "5d91d0f510ef77c1a90b3640795f38a40c3150b5bcb8029d64dc3f510f334d8f" diff --git a/pyproject.toml b/pyproject.toml index 795ff082..2831a1f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ pre-commit = "4.3.0" charset-normalizer = "3.4.6" python-discovery = "1.6.1" idna = "3.15" # dependency of requests -virtualenv = "21.14.9" +virtualenv = "21.14.13" certifi = "2026.10.23" filelock = "4.2.1" @@ -84,9 +84,9 @@ optional = true [tool.poetry.group.lint.dependencies] black = "26.10.1" -astroid = "3.3.9" +astroid = "4.3.4" isort = "5.13.2" -pylint = "3.3.9" +pylint = "4.1.3" mypy = "1.20.2" librt = "0.11.0" # mypy dependency boto3-stubs = {extras = ["essential"], version = "1.26.72"} @@ -157,8 +157,7 @@ ignore-patterns = ["^\\.#"] jobs = 1 limit-inference-results = 100 persistent = true -py-version = "3.10" -suggestion-mode = true +py-version = "3.11" [tool.pylint.basic] argument-naming-style = "snake_case" diff --git a/src/dve/core_engine/backends/base/reader.py b/src/dve/core_engine/backends/base/reader.py index eeeca7f4..97cc710a 100644 --- a/src/dve/core_engine/backends/base/reader.py +++ b/src/dve/core_engine/backends/base/reader.py @@ -165,9 +165,10 @@ def read_to_entity_type( entity=entity_name, record=None, failure_type="submission", - error_location=entity_name, error_code=config.error_code, error_message=config.error_message, + error_type="submission", + category="Empty entity", ) ], ) diff --git a/src/dve/core_engine/backends/exceptions.py b/src/dve/core_engine/backends/exceptions.py index 99808c8a..3bdc5760 100644 --- a/src/dve/core_engine/backends/exceptions.py +++ b/src/dve/core_engine/backends/exceptions.py @@ -79,7 +79,7 @@ def get_message_preamble(self) -> str: This will be joined to the location string with the joiner. """ - def get_joiner(self): + def get_joiner(self) -> str: """The joiner between the preamble and the context string.""" return "in" @@ -194,7 +194,7 @@ def get_message_epilogue(self) -> str: return f": {self.constraint}" -class ReaderErrorMixin(BackendErrorMixin): +class ReaderErrorMixin(BackendErrorMixin): # pylint: disable=W0223 """A mixin to be used by the reader errors.""" def get_joiner(self) -> str: @@ -233,33 +233,68 @@ def get_message_preamble(self) -> EntityName: return f"Refdata loader does not support reading refdata from {self.file_extension} files" -class EmptyFileError(ReaderErrorMixin, ValueError): +class EmptyFileError(ReaderErrorMixin, CriticalMessageBearingError): """The read file was empty.""" def get_message_preamble(self) -> str: return "File location provided is empty" + def __init__(self, *args: object, error_message: Optional[str] = None): + + super().__init__( + *args, + message=FeedbackMessage( + entity="Whole File", + record=None, + failure_type="submission", + error_code="EMPTYFILE", + error_message=error_message or self.get_message_preamble(), + ), + ) + -class MissingHeaderError(ReaderErrorMixin, ValueError): +class MissingHeaderError(ReaderErrorMixin, CriticalMessageBearingError): """A header is expected, but was not parsed.""" def get_message_preamble(self) -> str: return "No header was provided" + def __init__(self, *args: object, error_message: Optional[str] = None): + super().__init__( + *args, + message=FeedbackMessage( + entity="Whole File", + record=None, + failure_type="submission", + error_code="MISSINGHEADER", + error_message=error_message or self.get_message_preamble(), + ), + ) + -class FieldCountMismatch(ReaderErrorMixin, ValueError): +class FieldCountMismatch(ReaderErrorMixin, CriticalMessageBearingError): """An error raised when the number of expected fields does not match the number of parsed fields. """ def __init__(self, *args: object, n_expected_fields: int, n_actual_fields: int) -> None: - super().__init__(*args) self.n_expected_fields = n_expected_fields """The number of fields that should have been in the file.""" self.n_actual_fields = n_actual_fields """The number of fields that were actually in the file.""" + super().__init__( + *args, + message=FeedbackMessage( + entity="Whole File", + record=None, + failure_type="submission", + error_code="FIELDCOUNTMISMATCH", + error_message=self.get_message_preamble(), + ), + ) + def get_message_preamble(self) -> str: return ( f"Number of fields expected ({self.n_expected_fields}) does not " @@ -267,7 +302,7 @@ def get_message_preamble(self) -> str: ) -class SchemaMismatch(ReaderErrorMixin, ValueError): +class SchemaMismatch(ReaderErrorMixin, CriticalMessageBearingError): """An error raised when a file does not match a given schema (e.g. fields are missing). @@ -290,7 +325,16 @@ def __init__( tuple of actual and expected field type, both as strings). """ - super().__init__(*args) + super().__init__( + *args, + message=FeedbackMessage( + entity="Whole File", + record=None, + failure_type="submission", + error_code="SCHEMAMISMATCH", + error_message=self.get_message_preamble() + self.get_message_epilogue(), + ), + ) def get_message_preamble(self) -> str: return "Schema mismatch" diff --git a/src/dve/core_engine/backends/implementations/duckdb/contract.py b/src/dve/core_engine/backends/implementations/duckdb/contract.py index d9bb9bca..90bb02e6 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/contract.py +++ b/src/dve/core_engine/backends/implementations/duckdb/contract.py @@ -143,7 +143,7 @@ def apply_data_contract( if relation_is_empty(relation): self.logger.warning(f"+ Empty relation for {entity_name}") empty_df = pl.DataFrame([], schema=polars_schema) # type: ignore # pylint: disable=W0612 - relation = self._connection.sql("select * from empty_df") + entities[entity_name] = self._connection.sql("select * from empty_df") continue self.logger.info(f"+ Applying contract to: {entity_name}") diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py index 488f8c85..21671602 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py @@ -40,7 +40,7 @@ @duckdb_check_entity_empty @duckdb_record_index @duckdb_write_parquet -class DuckDBCSVReader(CSVFileReader): +class DuckDBCSVReader(CSVFileReader): # pylint: disable=W0223 """A reader for CSV files including the ability to compare the passed model to the file header, if it exists. @@ -99,7 +99,7 @@ def read_to_relation( # pylint: disable=unused-argument ) -> DuckDBPyRelation: """Returns a relation object from the source csv""" if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource} is empty.") + raise EmptyFileError(error_message=f"File at {resource} is empty.") if self.field_check: self.perform_field_check(resource, entity_name, schema, all_model_fields) @@ -137,7 +137,7 @@ def read_to_relation( # pylint: disable=unused-argument @polars_record_index -class PolarsToDuckDBCSVReader(DuckDBCSVReader): +class PolarsToDuckDBCSVReader(DuckDBCSVReader): # pylint: disable=W0223 """ Utilises the polars lazy csv reader which is then converted into a DuckDBPyRelation object. @@ -155,7 +155,7 @@ def read_to_relation( # pylint: disable=unused-argument ) -> DuckDBPyRelation: """Returns a relation object from the source csv""" if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource} is empty.") + raise EmptyFileError(error_message=f"File at {resource} is empty.") if self.field_check: self.perform_field_check(resource, entity_name, schema, all_model_fields) @@ -209,7 +209,7 @@ def read_to_relation( # pylint: disable=unused-argument return entity -class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader): +class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader): # pylint: disable=W0223 """A Reader for files with a `.csv` extension and where there are repeating "header" values within the file. Header in this case is not the column names at the top of a csv, rather a collection of unique records that would usually be structured in another entity. However, due diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py index 84b601de..2391e525 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py @@ -22,7 +22,7 @@ @duckdb_check_entity_empty @duckdb_record_index @duckdb_write_parquet -class DuckDBJSONReader(BaseFileReader): +class DuckDBJSONReader(BaseFileReader): # pylint: disable=W0223 """A reader for JSON files""" def __init__( diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py index 7e591e58..92da7e90 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py @@ -26,7 +26,7 @@ @duckdb_check_entity_empty @polars_record_index @duckdb_write_parquet -class DuckDBXMLStreamReader(XMLStreamReader): +class DuckDBXMLStreamReader(XMLStreamReader): # pylint: disable=W0223 """A reader for XML files""" def __init__(self, *, connection: Optional[DuckDBPyConnection] = None, **kwargs): diff --git a/src/dve/core_engine/backends/implementations/spark/contract.py b/src/dve/core_engine/backends/implementations/spark/contract.py index 432a7314..c32498fe 100644 --- a/src/dve/core_engine/backends/implementations/spark/contract.py +++ b/src/dve/core_engine/backends/implementations/spark/contract.py @@ -47,7 +47,7 @@ @spark_record_index @spark_write_parquet @spark_read_parquet -class SparkDataContract(BaseDataContract[DataFrame]): +class SparkDataContract(BaseDataContract[DataFrame]): # pylint: disable=W0223 """An implementation of a data contract in Spark. This uses Spark's RDD functionality to apply the data contract. diff --git a/src/dve/core_engine/backends/implementations/spark/readers/csv.py b/src/dve/core_engine/backends/implementations/spark/readers/csv.py index 6b79bb35..313f7079 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/csv.py @@ -24,7 +24,7 @@ @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkCSVReader(CSVFileReader): +class SparkCSVReader(CSVFileReader): # pylint: disable=W0223 """A Spark reader for CSV files.""" # pylint: disable=R0902 @@ -80,7 +80,7 @@ def read_to_dataframe( ) -> DataFrame: """Read a CSV file directly to a Spark DataFrame.""" if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource} is empty.") + raise EmptyFileError(error_message=f"File at {resource} is empty.") if self.field_check: self.perform_field_check(resource, entity_name, schema, all_model_fields) diff --git a/src/dve/core_engine/backends/implementations/spark/readers/json.py b/src/dve/core_engine/backends/implementations/spark/readers/json.py index 6404231c..e1747142 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/json.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/json.py @@ -22,7 +22,7 @@ @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkJSONReader(BaseFileReader): +class SparkJSONReader(BaseFileReader): # pylint: disable=W0223 """A Spark reader for JSON files.""" def __init__( @@ -60,7 +60,7 @@ def read_to_dataframe( ) -> DataFrame: """Read a JSON file directly to a Spark DataFrame.""" if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource} is empty.") + raise EmptyFileError(error_message=f"File at {resource} is empty.") spark_schema: StructType = get_type_from_annotation(schema) kwargs = { diff --git a/src/dve/core_engine/backends/implementations/spark/readers/xml.py b/src/dve/core_engine/backends/implementations/spark/readers/xml.py index 769c480e..da459290 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/xml.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/xml.py @@ -33,7 +33,7 @@ @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkXMLStreamReader(XMLStreamReader): +class SparkXMLStreamReader(XMLStreamReader): # pylint: disable=W0223 """An XML stream reader that adds a method to read to a dataframe""" spark = None @@ -61,7 +61,7 @@ def read_to_dataframe( @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkXMLReader(BasicXMLFileReader): # pylint: disable=too-many-instance-attributes +class SparkXMLReader(BasicXMLFileReader): # pylint: disable=too-many-instance-attributes,W0223 """A reader for XML files built atop Spark-XML.""" def __init__( @@ -130,7 +130,7 @@ def read_to_dataframe( """ if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource} is empty.") + raise EmptyFileError(error_message=f"File at {resource} is empty.") if self.xsd_location: msg = self._run_xmllint(file_uri=resource) diff --git a/src/dve/core_engine/backends/implementations/spark/rules.py b/src/dve/core_engine/backends/implementations/spark/rules.py index 8cdc8a1b..3d7958b7 100644 --- a/src/dve/core_engine/backends/implementations/spark/rules.py +++ b/src/dve/core_engine/backends/implementations/spark/rules.py @@ -61,7 +61,7 @@ @spark_write_parquet @spark_read_parquet @spark_filter_contract_errors -class SparkStepImplementations(BaseStepImplementations[DataFrame]): +class SparkStepImplementations(BaseStepImplementations[DataFrame]): # pylint: disable=W0223 """An implementation of transformation steps in Apache Spark.""" def __init__(self, spark_session: Optional[SparkSession] = None, **kwargs): diff --git a/src/dve/core_engine/backends/readers/csv.py b/src/dve/core_engine/backends/readers/csv.py index 619a3d19..48f153c0 100644 --- a/src/dve/core_engine/backends/readers/csv.py +++ b/src/dve/core_engine/backends/readers/csv.py @@ -28,7 +28,7 @@ from dve.parser.type_hints import URI -class CSVFileReader(BaseFileReader): +class CSVFileReader(BaseFileReader): # pylint: disable=W0223 """A base reader for CSV files.""" def __init__( @@ -130,7 +130,7 @@ def _parse_field_names(self, stream: IO[str]) -> list[str]: # can lead to null headers. parsed_field_names = reader.fieldnames if parsed_field_names is None: # pragma: no cover - raise MissingHeaderError("Header could not be parsed from CSV file") + raise MissingHeaderError(error_message="Header could not be parsed from CSV file") if not all( map(lambda string: isinstance(string, str), parsed_field_names) @@ -231,7 +231,7 @@ def read_to_py_iterator( ) -> Iterator[dict[str, Any]]: """Reads the data to an iterator of dictionaries""" if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource!r} is empty") + raise EmptyFileError(error_message=f"File at {resource!r} is empty") if self.field_check: self.perform_field_check(resource, entity_name, schema, all_model_fields) diff --git a/src/dve/core_engine/backends/readers/xml.py b/src/dve/core_engine/backends/readers/xml.py index 05167e18..fb584433 100644 --- a/src/dve/core_engine/backends/readers/xml.py +++ b/src/dve/core_engine/backends/readers/xml.py @@ -118,7 +118,7 @@ def clear(self) -> None: def __iter__(self) -> Iterator["XMLElement"]: ... -class BasicXMLFileReader(BaseFileReader): # pylint: disable=R0902 +class BasicXMLFileReader(BaseFileReader): # pylint: disable=R0902,W0223 """A reader for XML files built atop LXML.""" def __init__( @@ -319,7 +319,7 @@ def read_to_py_iterator( """ if get_content_length(resource) == 0: - raise EmptyFileError(f"File at {resource!r} is empty") + raise EmptyFileError(error_message=f"File at {resource!r} is empty") with open_stream(resource, "rb") as stream: for idx, record in enumerate(self._parse_xml(stream, schema), start=1): @@ -355,7 +355,7 @@ def write_parquet( # type: ignore return target_location -class XMLStreamReader(BasicXMLFileReader): +class XMLStreamReader(BasicXMLFileReader): # pylint: disable=W0223 """An XML parser which 'streams' the file, parsing only specific records. This means it requires more configuration, but should be much kinder on memory. diff --git a/src/dve/core_engine/exceptions.py b/src/dve/core_engine/exceptions.py index 4877baf1..6a5adcc2 100644 --- a/src/dve/core_engine/exceptions.py +++ b/src/dve/core_engine/exceptions.py @@ -13,7 +13,7 @@ def __init__( *args: object, messages: Optional[list[str]] = None, ) -> None: - super().__init__(error_message, *args) + super().__init__(*args) self.error_message = error_message """The error message explaining the critical processing error.""" self.messages = messages diff --git a/src/dve/metadata_parser/function_wrapper.py b/src/dve/metadata_parser/function_wrapper.py index cea12c03..76c0b0cd 100644 --- a/src/dve/metadata_parser/function_wrapper.py +++ b/src/dve/metadata_parser/function_wrapper.py @@ -180,12 +180,13 @@ def create_validator( return validator +# pylint: disable=unused-argument @pydantic.validate_call def _validator_kwargs( mode: str = "after", check_fields: bool | None = True, json_schema_input_type: Any = None, - **kwargs, # pylint: disable=unused-argument + **kwargs, ): return { "mode": mode, diff --git a/src/dve/parser/file_handling/implementations/file.py b/src/dve/parser/file_handling/implementations/file.py index 9fb29db0..597614b3 100644 --- a/src/dve/parser/file_handling/implementations/file.py +++ b/src/dve/parser/file_handling/implementations/file.py @@ -80,16 +80,14 @@ def put_byte_stream(self, byte_stream: IO[bytes], resource: URI): with path.open("wb") as file: shutil.copyfileobj(byte_stream, file) - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "write") def get_content_length(self, resource: URI) -> int: """Checks the size of the file""" try: return self._uri_to_path(resource).stat().st_size - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "read") def get_resource_exists(self, resource: URI) -> bool: @@ -97,8 +95,7 @@ def get_resource_exists(self, resource: URI) -> bool: try: local_path = self._uri_to_path(resource) return local_path.exists() and local_path.is_file() - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "read") def _iter_prefix_path(self, prefix_path: Path, recursive: bool = False) -> Iterator[Path]: @@ -120,8 +117,7 @@ def iter_prefix(self, prefix: URI, recursive: bool = False) -> Iterator[tuple[UR yield child_uri, "resource" else: yield child_uri + "/", "directory" - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, prefix, "list") def remove_resource(self, resource: URI): @@ -131,8 +127,7 @@ def remove_resource(self, resource: URI): if not path.exists(): return path.unlink() - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "delete") def remove_prefix(self, prefix: URI, recursive: bool = False): @@ -149,15 +144,13 @@ def remove_prefix(self, prefix: URI, recursive: bool = False): if recursive: shutil.rmtree(str(path)) return - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, prefix, "remove prefix") super().remove_prefix(prefix, recursive) try: path.rmdir() - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, prefix, "remove prefix") def _transfer_resource( diff --git a/src/dve/parser/file_handling/log_handler.py b/src/dve/parser/file_handling/log_handler.py index 1b406052..cd8d4ae5 100644 --- a/src/dve/parser/file_handling/log_handler.py +++ b/src/dve/parser/file_handling/log_handler.py @@ -154,8 +154,7 @@ def emit(self, record: logging.LogRecord): msg = self.format(record) self._stream.write(msg + "\n") self._stream.flush() - # pylint: disable=bare-except - except: # noqa: E722 pragma: no cover + except Exception: # noqa: E722 pragma: no cover pylint: disable=bare-except,W0718 self.handleError(record) finally: self.release() diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py index 53185a06..2e499fc4 100644 --- a/src/dve/pipeline/pipeline.py +++ b/src/dve/pipeline/pipeline.py @@ -661,6 +661,7 @@ def apply_business_rules( # pylint: disable=R0914,R0915 submission_status.validation_failed = True elif not rules_success: submission_status.processing_failed = True + return submission_info, submission_status for entity_name, entity in entity_manager.entities.items(): # Note BI filtering done within the apply_rules @@ -930,6 +931,7 @@ def _get_error_dataframes(self, submission_id: str, include_record_rejections: b pl.col(column).cast(ERROR_SCHEMA[column]) # type: ignore for column in df.columns ) + df = df.with_columns(pl.col("Data_Item").fill_null("")) df = df.sort("Type", descending=False) # type: ignore errors_dfs.append(df)