diff --git a/poetry.lock b/poetry.lock index aa6c10aa..861553ba 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -29,14 +29,14 @@ test = ["coverage", "mypy", "pexpect", "ruff", "wheel"] [[package]] name = "astroid" -version = "3.3.9" +version = "4.3.4" description = "An abstract syntax tree for Python with inference support." optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["dev", "lint"] files = [ - {file = "astroid-3.3.9-py3-none-any.whl", hash = "sha256:d05bfd0acba96a7bd43e222828b7d9bc1e138aaeb0649707908d3702a9831248"}, - {file = "astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550"}, + {file = "astroid-4.3.4-py3-none-any.whl", hash = "sha256:2bcd0d02648a443a4b818c952c3550091989daefac3c12d3b83b2289482e0818"}, + {file = "astroid-4.3.4.tar.gz", hash = "sha256:d515a105722b72098bbe82d430d65e635f742b6cbac3bdfaf8b7c188b87c5e39"}, ] [[package]] @@ -519,7 +519,7 @@ files = [ [package.dependencies] jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" -urllib3 = {version = ">=1.25.4,!=2.2.0,<3", markers = "python_version >= \"3.10\""} +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} [package.extras] crt = ["awscrt (==0.22.0)"] @@ -1875,7 +1875,7 @@ python-dateutil = ">=2.1,<3.0.0" PyYAML = {version = ">=5.1", optional = true, markers = "extra == \"s3\""} requests = ">=2.5" responses = ">=0.13.0" -werkzeug = ">=0.5,!=2.2.0,!=2.2.1" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" xmltodict = "*" [package.extras] @@ -2684,24 +2684,24 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pylint" -version = "3.3.9" +version = "4.1.3" description = "python code static checker" optional = false -python-versions = ">=3.9.0" +python-versions = ">=3.10.0" groups = ["dev", "lint"] files = [ - {file = "pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7"}, - {file = "pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a"}, + {file = "pylint-4.1.3-py3-none-any.whl", hash = "sha256:a85357cae24f33ad8d86c8f3daaa92c600ae4012b54a57299cee76000e9364cf"}, + {file = "pylint-4.1.3.tar.gz", hash = "sha256:9928603068edfa0d1a3c167f174b099d4b97c3db75d32d0fcdd029770b4713a9"}, ] [package.dependencies] -astroid = ">=3.3.8,<=3.4.0.dev0" +astroid = ">=4.3.4,<=4.4" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = [ {version = ">=0.3.6", markers = "python_version == \"3.11\""}, {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, ] -isort = ">=4.2.5,!=5.13,<7" +isort = ">=5,<5.13 || >5.13,<10" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2" tomlkit = ">=0.10.1" @@ -3596,4 +3596,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "f27ecec1d155a999f58619d0529f6a8aa33d95f5070ef3708aa0e46a8aa88f99" +content-hash = "5d91d0f510ef77c1a90b3640795f38a40c3150b5bcb8029d64dc3f510f334d8f" diff --git a/pyproject.toml b/pyproject.toml index 4966b7e1..2831a1f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,9 +84,9 @@ optional = true [tool.poetry.group.lint.dependencies] black = "26.10.1" -astroid = "3.3.9" +astroid = "4.3.4" isort = "5.13.2" -pylint = "3.3.9" +pylint = "4.1.3" mypy = "1.20.2" librt = "0.11.0" # mypy dependency boto3-stubs = {extras = ["essential"], version = "1.26.72"} @@ -157,8 +157,7 @@ ignore-patterns = ["^\\.#"] jobs = 1 limit-inference-results = 100 persistent = true -py-version = "3.10" -suggestion-mode = true +py-version = "3.11" [tool.pylint.basic] argument-naming-style = "snake_case" diff --git a/src/dve/core_engine/backends/exceptions.py b/src/dve/core_engine/backends/exceptions.py index 99808c8a..3878853d 100644 --- a/src/dve/core_engine/backends/exceptions.py +++ b/src/dve/core_engine/backends/exceptions.py @@ -79,7 +79,7 @@ def get_message_preamble(self) -> str: This will be joined to the location string with the joiner. """ - def get_joiner(self): + def get_joiner(self) -> str: """The joiner between the preamble and the context string.""" return "in" @@ -194,7 +194,7 @@ def get_message_epilogue(self) -> str: return f": {self.constraint}" -class ReaderErrorMixin(BackendErrorMixin): +class ReaderErrorMixin(BackendErrorMixin): # pylint: disable=W0223 """A mixin to be used by the reader errors.""" def get_joiner(self) -> str: diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py index 488f8c85..9064816b 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py @@ -40,7 +40,7 @@ @duckdb_check_entity_empty @duckdb_record_index @duckdb_write_parquet -class DuckDBCSVReader(CSVFileReader): +class DuckDBCSVReader(CSVFileReader): # pylint: disable=W0223 """A reader for CSV files including the ability to compare the passed model to the file header, if it exists. @@ -137,7 +137,7 @@ def read_to_relation( # pylint: disable=unused-argument @polars_record_index -class PolarsToDuckDBCSVReader(DuckDBCSVReader): +class PolarsToDuckDBCSVReader(DuckDBCSVReader): # pylint: disable=W0223 """ Utilises the polars lazy csv reader which is then converted into a DuckDBPyRelation object. @@ -209,7 +209,7 @@ def read_to_relation( # pylint: disable=unused-argument return entity -class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader): +class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader): # pylint: disable=W0223 """A Reader for files with a `.csv` extension and where there are repeating "header" values within the file. Header in this case is not the column names at the top of a csv, rather a collection of unique records that would usually be structured in another entity. However, due diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py index 84b601de..2391e525 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/json.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/json.py @@ -22,7 +22,7 @@ @duckdb_check_entity_empty @duckdb_record_index @duckdb_write_parquet -class DuckDBJSONReader(BaseFileReader): +class DuckDBJSONReader(BaseFileReader): # pylint: disable=W0223 """A reader for JSON files""" def __init__( diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py index 7e591e58..92da7e90 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py +++ b/src/dve/core_engine/backends/implementations/duckdb/readers/xml.py @@ -26,7 +26,7 @@ @duckdb_check_entity_empty @polars_record_index @duckdb_write_parquet -class DuckDBXMLStreamReader(XMLStreamReader): +class DuckDBXMLStreamReader(XMLStreamReader): # pylint: disable=W0223 """A reader for XML files""" def __init__(self, *, connection: Optional[DuckDBPyConnection] = None, **kwargs): diff --git a/src/dve/core_engine/backends/implementations/spark/contract.py b/src/dve/core_engine/backends/implementations/spark/contract.py index 432a7314..c32498fe 100644 --- a/src/dve/core_engine/backends/implementations/spark/contract.py +++ b/src/dve/core_engine/backends/implementations/spark/contract.py @@ -47,7 +47,7 @@ @spark_record_index @spark_write_parquet @spark_read_parquet -class SparkDataContract(BaseDataContract[DataFrame]): +class SparkDataContract(BaseDataContract[DataFrame]): # pylint: disable=W0223 """An implementation of a data contract in Spark. This uses Spark's RDD functionality to apply the data contract. diff --git a/src/dve/core_engine/backends/implementations/spark/readers/csv.py b/src/dve/core_engine/backends/implementations/spark/readers/csv.py index 6b79bb35..08142a69 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/csv.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/csv.py @@ -24,7 +24,7 @@ @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkCSVReader(CSVFileReader): +class SparkCSVReader(CSVFileReader): # pylint: disable=W0223 """A Spark reader for CSV files.""" # pylint: disable=R0902 diff --git a/src/dve/core_engine/backends/implementations/spark/readers/json.py b/src/dve/core_engine/backends/implementations/spark/readers/json.py index 6404231c..1d20b63e 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/json.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/json.py @@ -22,7 +22,7 @@ @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkJSONReader(BaseFileReader): +class SparkJSONReader(BaseFileReader): # pylint: disable=W0223 """A Spark reader for JSON files.""" def __init__( diff --git a/src/dve/core_engine/backends/implementations/spark/readers/xml.py b/src/dve/core_engine/backends/implementations/spark/readers/xml.py index 769c480e..d5589466 100644 --- a/src/dve/core_engine/backends/implementations/spark/readers/xml.py +++ b/src/dve/core_engine/backends/implementations/spark/readers/xml.py @@ -33,7 +33,7 @@ @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkXMLStreamReader(XMLStreamReader): +class SparkXMLStreamReader(XMLStreamReader): # pylint: disable=W0223 """An XML stream reader that adds a method to read to a dataframe""" spark = None @@ -61,7 +61,7 @@ def read_to_dataframe( @spark_check_entity_empty @spark_record_index @spark_write_parquet -class SparkXMLReader(BasicXMLFileReader): # pylint: disable=too-many-instance-attributes +class SparkXMLReader(BasicXMLFileReader): # pylint: disable=too-many-instance-attributes,W0223 """A reader for XML files built atop Spark-XML.""" def __init__( diff --git a/src/dve/core_engine/backends/implementations/spark/rules.py b/src/dve/core_engine/backends/implementations/spark/rules.py index 8cdc8a1b..3d7958b7 100644 --- a/src/dve/core_engine/backends/implementations/spark/rules.py +++ b/src/dve/core_engine/backends/implementations/spark/rules.py @@ -61,7 +61,7 @@ @spark_write_parquet @spark_read_parquet @spark_filter_contract_errors -class SparkStepImplementations(BaseStepImplementations[DataFrame]): +class SparkStepImplementations(BaseStepImplementations[DataFrame]): # pylint: disable=W0223 """An implementation of transformation steps in Apache Spark.""" def __init__(self, spark_session: Optional[SparkSession] = None, **kwargs): diff --git a/src/dve/core_engine/backends/readers/csv.py b/src/dve/core_engine/backends/readers/csv.py index 619a3d19..903740fa 100644 --- a/src/dve/core_engine/backends/readers/csv.py +++ b/src/dve/core_engine/backends/readers/csv.py @@ -28,7 +28,7 @@ from dve.parser.type_hints import URI -class CSVFileReader(BaseFileReader): +class CSVFileReader(BaseFileReader): # pylint: disable=W0223 """A base reader for CSV files.""" def __init__( diff --git a/src/dve/core_engine/backends/readers/xml.py b/src/dve/core_engine/backends/readers/xml.py index 05167e18..71df39b1 100644 --- a/src/dve/core_engine/backends/readers/xml.py +++ b/src/dve/core_engine/backends/readers/xml.py @@ -118,7 +118,7 @@ def clear(self) -> None: def __iter__(self) -> Iterator["XMLElement"]: ... -class BasicXMLFileReader(BaseFileReader): # pylint: disable=R0902 +class BasicXMLFileReader(BaseFileReader): # pylint: disable=R0902,W0223 """A reader for XML files built atop LXML.""" def __init__( @@ -355,7 +355,7 @@ def write_parquet( # type: ignore return target_location -class XMLStreamReader(BasicXMLFileReader): +class XMLStreamReader(BasicXMLFileReader): # pylint: disable=W0223 """An XML parser which 'streams' the file, parsing only specific records. This means it requires more configuration, but should be much kinder on memory. diff --git a/src/dve/metadata_parser/function_wrapper.py b/src/dve/metadata_parser/function_wrapper.py index cea12c03..76c0b0cd 100644 --- a/src/dve/metadata_parser/function_wrapper.py +++ b/src/dve/metadata_parser/function_wrapper.py @@ -180,12 +180,13 @@ def create_validator( return validator +# pylint: disable=unused-argument @pydantic.validate_call def _validator_kwargs( mode: str = "after", check_fields: bool | None = True, json_schema_input_type: Any = None, - **kwargs, # pylint: disable=unused-argument + **kwargs, ): return { "mode": mode, diff --git a/src/dve/parser/file_handling/implementations/file.py b/src/dve/parser/file_handling/implementations/file.py index 9fb29db0..597614b3 100644 --- a/src/dve/parser/file_handling/implementations/file.py +++ b/src/dve/parser/file_handling/implementations/file.py @@ -80,16 +80,14 @@ def put_byte_stream(self, byte_stream: IO[bytes], resource: URI): with path.open("wb") as file: shutil.copyfileobj(byte_stream, file) - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "write") def get_content_length(self, resource: URI) -> int: """Checks the size of the file""" try: return self._uri_to_path(resource).stat().st_size - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "read") def get_resource_exists(self, resource: URI) -> bool: @@ -97,8 +95,7 @@ def get_resource_exists(self, resource: URI) -> bool: try: local_path = self._uri_to_path(resource) return local_path.exists() and local_path.is_file() - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "read") def _iter_prefix_path(self, prefix_path: Path, recursive: bool = False) -> Iterator[Path]: @@ -120,8 +117,7 @@ def iter_prefix(self, prefix: URI, recursive: bool = False) -> Iterator[tuple[UR yield child_uri, "resource" else: yield child_uri + "/", "directory" - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, prefix, "list") def remove_resource(self, resource: URI): @@ -131,8 +127,7 @@ def remove_resource(self, resource: URI): if not path.exists(): return path.unlink() - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, resource, "delete") def remove_prefix(self, prefix: URI, recursive: bool = False): @@ -149,15 +144,13 @@ def remove_prefix(self, prefix: URI, recursive: bool = False): if recursive: shutil.rmtree(str(path)) return - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, prefix, "remove prefix") super().remove_prefix(prefix, recursive) try: path.rmdir() - # pylint: disable=broad-except - except Exception as err: # pragma: no cover + except Exception as err: # pragma: no cover pylint: disable=broad-except self._handle_error(err, prefix, "remove prefix") def _transfer_resource( diff --git a/src/dve/parser/file_handling/log_handler.py b/src/dve/parser/file_handling/log_handler.py index 1b406052..cd8d4ae5 100644 --- a/src/dve/parser/file_handling/log_handler.py +++ b/src/dve/parser/file_handling/log_handler.py @@ -154,8 +154,7 @@ def emit(self, record: logging.LogRecord): msg = self.format(record) self._stream.write(msg + "\n") self._stream.flush() - # pylint: disable=bare-except - except: # noqa: E722 pragma: no cover + except Exception: # noqa: E722 pragma: no cover pylint: disable=bare-except,W0718 self.handleError(record) finally: self.release()