build: update pylint

pyproject.toml

@@ -84,9 +84,9 @@ optional = true
 
 [tool.poetry.group.lint.dependencies]
 black = "26.10.1"
-astroid = "3.3.9"
+astroid = "4.3.4"
 isort = "5.13.2"
-pylint = "3.3.9"
+pylint = "4.1.3"
 mypy = "1.20.2"
 librt = "0.11.0"  # mypy dependency
 boto3-stubs = {extras = ["essential"], version = "1.26.72"}
@@ -157,8 +157,7 @@ ignore-patterns = ["^\\.#"]
 jobs = 1
 limit-inference-results = 100
 persistent = true
-py-version = "3.10"
-suggestion-mode = true
+py-version = "3.11"
 
 [tool.pylint.basic]
 argument-naming-style = "snake_case"

src/dve/core_engine/backends/exceptions.py

@@ -79,7 +79,7 @@ def get_message_preamble(self) -> str:
         This will be joined to the location string with the joiner.
         """
 
-    def get_joiner(self):
+    def get_joiner(self) -> str:
         """The joiner between the preamble and the context string."""
         return "in"
 
@@ -194,7 +194,7 @@ def get_message_epilogue(self) -> str:
         return f": {self.constraint}"
 
 
-class ReaderErrorMixin(BackendErrorMixin):
+class ReaderErrorMixin(BackendErrorMixin):  # pylint: disable=W0223
     """A mixin to be used by the reader errors."""
 
     def get_joiner(self) -> str:

src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -40,7 +40,7 @@
 @duckdb_check_entity_empty
 @duckdb_record_index
 @duckdb_write_parquet
-class DuckDBCSVReader(CSVFileReader):
+class DuckDBCSVReader(CSVFileReader):  # pylint: disable=W0223
     """A reader for CSV files including the ability to compare the passed model
     to the file header, if it exists.
 
@@ -137,7 +137,7 @@ def read_to_relation(  # pylint: disable=unused-argument
 
 
 @polars_record_index
-class PolarsToDuckDBCSVReader(DuckDBCSVReader):
+class PolarsToDuckDBCSVReader(DuckDBCSVReader):  # pylint: disable=W0223
     """
     Utilises the polars lazy csv reader which is then converted into a DuckDBPyRelation object.
 
@@ -209,7 +209,7 @@ def read_to_relation(  # pylint: disable=unused-argument
         return entity
 
 
-class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):
+class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):  # pylint: disable=W0223
     """A Reader for files with a `.csv` extension and where there are repeating "header" values
     within the file. Header in this case is not the column names at the top of a csv, rather a
     collection of unique records that would usually be structured in another entity. However, due

src/dve/core_engine/backends/implementations/duckdb/readers/json.py

@@ -22,7 +22,7 @@
 @duckdb_check_entity_empty
 @duckdb_record_index
 @duckdb_write_parquet
-class DuckDBJSONReader(BaseFileReader):
+class DuckDBJSONReader(BaseFileReader):  # pylint: disable=W0223
     """A reader for JSON files"""
 
     def __init__(

src/dve/core_engine/backends/implementations/duckdb/readers/xml.py

@@ -26,7 +26,7 @@
 @duckdb_check_entity_empty
 @polars_record_index
 @duckdb_write_parquet
-class DuckDBXMLStreamReader(XMLStreamReader):
+class DuckDBXMLStreamReader(XMLStreamReader):  # pylint: disable=W0223
     """A reader for XML files"""
 
     def __init__(self, *, connection: Optional[DuckDBPyConnection] = None, **kwargs):

src/dve/core_engine/backends/implementations/spark/contract.py

@@ -47,7 +47,7 @@
 @spark_record_index
 @spark_write_parquet
 @spark_read_parquet
-class SparkDataContract(BaseDataContract[DataFrame]):
+class SparkDataContract(BaseDataContract[DataFrame]):  # pylint: disable=W0223
     """An implementation of a data contract in Spark.
 
     This uses Spark's RDD functionality to apply the data contract.

src/dve/core_engine/backends/implementations/spark/readers/csv.py

@@ -24,7 +24,7 @@
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkCSVReader(CSVFileReader):
+class SparkCSVReader(CSVFileReader):  # pylint: disable=W0223
     """A Spark reader for CSV files."""
 
     # pylint: disable=R0902

src/dve/core_engine/backends/implementations/spark/readers/json.py

@@ -22,7 +22,7 @@
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkJSONReader(BaseFileReader):
+class SparkJSONReader(BaseFileReader):  # pylint: disable=W0223
     """A Spark reader for JSON files."""
 
     def __init__(

src/dve/core_engine/backends/implementations/spark/readers/xml.py

@@ -33,7 +33,7 @@
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkXMLStreamReader(XMLStreamReader):
+class SparkXMLStreamReader(XMLStreamReader):  # pylint: disable=W0223
     """An XML stream reader that adds a method to read to a dataframe"""
 
     spark = None
@@ -61,7 +61,7 @@ def read_to_dataframe(
 @spark_check_entity_empty
 @spark_record_index
 @spark_write_parquet
-class SparkXMLReader(BasicXMLFileReader):  # pylint: disable=too-many-instance-attributes
+class SparkXMLReader(BasicXMLFileReader):  # pylint: disable=too-many-instance-attributes,W0223
     """A reader for XML files built atop Spark-XML."""
 
     def __init__(

src/dve/core_engine/backends/implementations/spark/rules.py

@@ -61,7 +61,7 @@
 @spark_write_parquet
 @spark_read_parquet
 @spark_filter_contract_errors
-class SparkStepImplementations(BaseStepImplementations[DataFrame]):
+class SparkStepImplementations(BaseStepImplementations[DataFrame]):  # pylint: disable=W0223
     """An implementation of transformation steps in Apache Spark."""
 
     def __init__(self, spark_session: Optional[SparkSession] = None, **kwargs):

src/dve/core_engine/backends/readers/csv.py

@@ -28,7 +28,7 @@
 from dve.parser.type_hints import URI
 
 
-class CSVFileReader(BaseFileReader):
+class CSVFileReader(BaseFileReader):  # pylint: disable=W0223
     """A base reader for CSV files."""
 
     def __init__(

src/dve/core_engine/backends/readers/xml.py

@@ -118,7 +118,7 @@ def clear(self) -> None:
     def __iter__(self) -> Iterator["XMLElement"]: ...
 
 
-class BasicXMLFileReader(BaseFileReader):  # pylint: disable=R0902
+class BasicXMLFileReader(BaseFileReader):  # pylint: disable=R0902,W0223
     """A reader for XML files built atop LXML."""
 
     def __init__(
@@ -355,7 +355,7 @@ def write_parquet(  # type: ignore
         return target_location
 
 
-class XMLStreamReader(BasicXMLFileReader):
+class XMLStreamReader(BasicXMLFileReader):  # pylint: disable=W0223
     """An XML parser which 'streams' the file, parsing only specific records.
     This means it requires more configuration, but should be much kinder on memory.
 

src/dve/metadata_parser/function_wrapper.py

@@ -180,12 +180,13 @@ def create_validator(
     return validator
 
 
+# pylint: disable=unused-argument
 @pydantic.validate_call
 def _validator_kwargs(
     mode: str = "after",
     check_fields: bool | None = True,
     json_schema_input_type: Any = None,
-    **kwargs,  # pylint: disable=unused-argument
+    **kwargs,
 ):
     return {
         "mode": mode,

src/dve/parser/file_handling/implementations/file.py

@@ -80,25 +80,22 @@ def put_byte_stream(self, byte_stream: IO[bytes], resource: URI):
 
             with path.open("wb") as file:
                 shutil.copyfileobj(byte_stream, file)
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, resource, "write")
 
     def get_content_length(self, resource: URI) -> int:
         """Checks the size of the file"""
         try:
             return self._uri_to_path(resource).stat().st_size
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, resource, "read")
 
     def get_resource_exists(self, resource: URI) -> bool:
         """Checks if a resource exists at the given uri"""
         try:
             local_path = self._uri_to_path(resource)
             return local_path.exists() and local_path.is_file()
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, resource, "read")
 
     def _iter_prefix_path(self, prefix_path: Path, recursive: bool = False) -> Iterator[Path]:
@@ -120,8 +117,7 @@ def iter_prefix(self, prefix: URI, recursive: bool = False) -> Iterator[tuple[UR
                     yield child_uri, "resource"
                 else:
                     yield child_uri + "/", "directory"
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, prefix, "list")
 
     def remove_resource(self, resource: URI):
@@ -131,8 +127,7 @@ def remove_resource(self, resource: URI):
             if not path.exists():
                 return
             path.unlink()
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, resource, "delete")
 
     def remove_prefix(self, prefix: URI, recursive: bool = False):
@@ -149,15 +144,13 @@ def remove_prefix(self, prefix: URI, recursive: bool = False):
             if recursive:
                 shutil.rmtree(str(path))
                 return
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, prefix, "remove prefix")
 
         super().remove_prefix(prefix, recursive)
         try:
             path.rmdir()
-        # pylint: disable=broad-except
-        except Exception as err:  # pragma: no cover
+        except Exception as err:  # pragma: no cover pylint: disable=broad-except
             self._handle_error(err, prefix, "remove prefix")
 
     def _transfer_resource(

src/dve/parser/file_handling/log_handler.py

@@ -154,8 +154,7 @@
                 msg = self.format(record)
                 self._stream.write(msg + "\n")
                 self._stream.flush()
-            # pylint: disable=bare-except
-            except:  # noqa: E722 pragma: no cover
+            except Exception:  # noqa: E722 pragma: no cover pylint: disable=bare-except,W0718
                 self.handleError(record)
         finally:
             self.release()