Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 1 addition & 6 deletions ext/MatrixAlgebraKitAMDGPUExt/MatrixAlgebraKitAMDGPUExt.jl
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ using MatrixAlgebraKit: default_qr_algorithm, default_lq_algorithm, default_svd_
import MatrixAlgebraKit: geqrf!, ungqr!, unmqr!, gesvd!, gesdd!, gesvdx!, gesvdj!
import MatrixAlgebraKit: gesvdj_batched!, gesdd_batched!, gesvd_batched!, gesvdx_batched!
import MatrixAlgebraKit: heevj!, heevd!, heev!, heevx!
import MatrixAlgebraKit: _sylvester, svd_rank, svd_pullback!
import MatrixAlgebraKit: svd_rank, svd_pullback!
using AMDGPU
using LinearAlgebra
using LinearAlgebra: BlasFloat
Expand Down Expand Up @@ -165,9 +165,4 @@ function MatrixAlgebraKit._mul_herm!(C::StridedROCMatrix{T}, A::StridedROCMatrix
return C
end

function _sylvester(A::AnyROCMatrix, B::AnyROCMatrix, C::AnyROCMatrix)
hX = sylvester(collect(A), collect(B), collect(C))
return ROCArray(hX)
end

end
9 changes: 1 addition & 8 deletions ext/MatrixAlgebraKitCUDAExt/MatrixAlgebraKitCUDAExt.jl
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ using MatrixAlgebraKit: default_qr_algorithm, default_lq_algorithm, default_svd_
import MatrixAlgebraKit: geqrf!, ungqr!, unmqr!, gesvd!, gesvdp!, gesvdr!, gesvdj!
import MatrixAlgebraKit: gesvdj_batched!
import MatrixAlgebraKit: heevj!, heevd!, geev!
import MatrixAlgebraKit: _gpu_Xgesvdr!, _sylvester
import MatrixAlgebraKit: _gpu_Xgesvdr!
using CUDA, CUDA.cuBLAS
using CUDA: i32
using LinearAlgebra
Expand Down Expand Up @@ -88,11 +88,4 @@ function MatrixAlgebraKit._mul_herm!(C::StridedCuMatrix{T}, A::StridedCuMatrix{T
return C
end

function _sylvester(A::AnyCuMatrix, B::AnyCuMatrix, C::AnyCuMatrix)
# https://github.com/JuliaGPU/CUDA.jl/issues/3021
# to add native sylvester to CUDA
hX = sylvester(collect(A), collect(B), collect(C))
return CuArray(hX)
end

end
2 changes: 1 addition & 1 deletion src/MatrixAlgebraKit.jl
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module MatrixAlgebraKit
using LinearAlgebra: LinearAlgebra
using LinearAlgebra: norm # TODO: eleminate if we use VectorInterface.jl?
using LinearAlgebra: mul!, rmul!, lmul!, adjoint!, rdiv!, ldiv!
using LinearAlgebra: sylvester, lu!, diagm
using LinearAlgebra: lu!, diagm
using LinearAlgebra: isposdef, issymmetric
using LinearAlgebra: Diagonal, Hermitian, diag, diagind, isdiag
using LinearAlgebra: UpperTriangular, LowerTriangular
Expand Down
21 changes: 19 additions & 2 deletions src/common/pullbacks.jl
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,25 @@ function iszerotangent end
iszerotangent(::Any) = false
iszerotangent(::Nothing) = true

# fallback
_sylvester(A, B, C) = LinearAlgebra.sylvester(A, B, C)
# Solve the Sylvester equation `P * X + X * P = C` for a Hermitian positive semidefinite `P`
# with eigendecomposition `P = U * Diagonal(d) * U'`.
function _sylvester_psd(U, d, C, tol)
Y = U' * C * U
dd = d .+ transpose(d)
Δ = maximum(abs, (abs.(dd) .< tol) .* Y; init = zero(real(eltype(Y))))
Y .*= inv_safe.(dd, tol)
return U * Y * U', Δ
end

# Remove from `X` its block within the null space of the Hermitian positive semidefinite
# `P = U * Diagonal(d) * U'`, i.e. the eigenvectors with eigenvalues smaller than `tol`.
function _remove_null_block!(X, U, d, tol)
isnull = abs.(d) .< tol
any(isnull) || return X
Y = U' * X * U
Y .*= isnull .& transpose(isnull)
return mul!(X, U, Y * U', -1, 1)
end

"""
select_indices(r::AbstractRange, ind)
Expand Down
72 changes: 57 additions & 15 deletions src/pullbacks/polar.jl
Original file line number Diff line number Diff line change
@@ -1,59 +1,101 @@
"""
left_polar_pullback!(ΔA, A, WP, ΔWP)
left_polar_pullback!(
ΔA, A, WP, ΔWP;
rank_atol::Real = default_pullback_rank_atol(WP[2]),
gauge_atol::Real = default_pullback_gauge_atol(ΔWP...)
)

Adds the pullback from the left polar decomposition of `A` to `ΔA` given the output `WP` and
cotangent `ΔWP` of `left_polar(A)`.

Eigenvalues of `P` smaller than `rank_atol` are treated as zero, in which case `A` is rank
deficient and `W` is not uniquely determined on the null space of `P`. Components of `ΔW`
along this null space are discarded. The block of `ΔP` within the null space, where `P`
is not differentiable, is discarded as well.
"""
function left_polar_pullback!(ΔA::AbstractMatrix, A, WP, ΔWP; kwargs...)
function left_polar_pullback!(
ΔA::AbstractMatrix, A, WP, ΔWP;
rank_atol::Real = default_pullback_rank_atol(WP[2]),
gauge_atol::Real = default_pullback_gauge_atol(ΔWP...)
)
# Extract the Polar components
W, P = WP

# Extract and check the cotangents
ΔW, ΔP = ΔWP
D, U = eigh_full(P)
d = diagview(D)
if !iszerotangent(ΔP)
ΔP = project_hermitian(ΔP)
ΔP = _remove_null_block!(ΔP, U, d, rank_atol)
end
M = zero(P)
!iszerotangent(ΔW) && mul!(M, W', ΔW, 1, 1)
!iszerotangent(ΔP) && mul!(M, ΔP, P, -1, 1)
C = _sylvester(P, P, M' - M)
C, Δgauge = _sylvester_psd(U, d, M - M', rank_atol)
!iszerotangent(ΔP) && (C .+= ΔP)
ΔA = mul!(ΔA, W, C, 1, 1)
if !iszerotangent(ΔW)
ΔWP = ΔW / P
WdΔWP = W' * ΔWP
ΔWP = mul!(ΔWP, W, WdΔWP, -1, 1)
ΔA .+= ΔWP
# (1 - W * W') * ΔW * pinv(P)
ΔWU = ΔW * U
WᴴΔWU = W' * ΔWU
ΔWU = mul!(ΔWU, W, WᴴΔWU, -1, 1)
Δgauge = max(Δgauge, maximum(abs, (abs.(transpose(d)) .< rank_atol) .* ΔWU; init = zero(real(eltype(ΔWU)))))
ΔWU .*= transpose(inv_safe.(d, rank_atol))
ΔA = mul!(ΔA, ΔWU, U', 1, 1)
end
Δgauge ≤ gauge_atol ||
@warn "`left_polar` cotangents sensitive to gauge choice: (|Δgauge| = $Δgauge)"
return ΔA
end

"""
right_polar_pullback!(ΔA, A, PWᴴ, ΔPWᴴ)
right_polar_pullback!(
ΔA, A, PWᴴ, ΔPWᴴ;
rank_atol::Real = default_pullback_rank_atol(PWᴴ[1]),
gauge_atol::Real = default_pullback_gauge_atol(ΔPWᴴ...)
)

Adds the pullback from the left polar decomposition of `A` to `ΔA` given the output `PWᴴ`
Adds the pullback from the right polar decomposition of `A` to `ΔA` given the output `PWᴴ`
and cotangent `ΔPWᴴ` of `right_polar(A)`.

Eigenvalues of `P` smaller than `rank_atol` are treated as zero, in which case `A` is rank
deficient and `Wᴴ` is not uniquely determined on the null space of `P`. Components of `ΔWᴴ`
along this null space are discarded. The block of `ΔP` within the null space, where `P` is
not differentiable, is discarded as well.
"""
function right_polar_pullback!(ΔA::AbstractMatrix, A, PWᴴ, ΔPWᴴ; kwargs...)
function right_polar_pullback!(
ΔA::AbstractMatrix, A, PWᴴ, ΔPWᴴ;
rank_atol::Real = default_pullback_rank_atol(PWᴴ[1]),
gauge_atol::Real = default_pullback_gauge_atol(ΔPWᴴ...)
)
# Extract the Polar components
P, Wᴴ = PWᴴ

# Extract and check the cotangents
ΔP, ΔWᴴ = ΔPWᴴ
D, U = eigh_full(P)
d = diagview(D)
if !iszerotangent(ΔP)
ΔP = project_hermitian(ΔP)
ΔP = _remove_null_block!(ΔP, U, d, rank_atol)
end
M = zero(P)
!iszerotangent(ΔWᴴ) && mul!(M, ΔWᴴ, Wᴴ', 1, 1)
!iszerotangent(ΔP) && mul!(M, P, ΔP, -1, 1)
C = _sylvester(P, P, M' - M)
C, Δgauge = _sylvester_psd(U, d, M - M', rank_atol)
!iszerotangent(ΔP) && (C .+= ΔP)
ΔA = mul!(ΔA, C, Wᴴ, 1, 1)
if !iszerotangent(ΔWᴴ)
PΔWᴴ = P \ ΔWᴴ
PΔWᴴW = PΔWᴴ * Wᴴ'
PΔWᴴ = mul!(PΔWᴴ, PΔWᴴW, Wᴴ, -1, 1)
ΔA .+= PΔWᴴ
# pinv(P) * ΔWᴴ * (1 - Wᴴ' * Wᴴ)
UᴴΔWᴴ = U' * ΔWᴴ
UᴴΔWᴴW = UᴴΔWᴴ * Wᴴ'
UᴴΔWᴴ = mul!(UᴴΔWᴴ, UᴴΔWᴴW, Wᴴ, -1, 1)
Δgauge = max(Δgauge, maximum(abs, (abs.(d) .< rank_atol) .* UᴴΔWᴴ; init = zero(real(eltype(UᴴΔWᴴ)))))
UᴴΔWᴴ .*= inv_safe.(d, rank_atol)
ΔA = mul!(ΔA, U, UᴴΔWᴴ, 1, 1)
end
Δgauge ≤ gauge_atol ||
@warn "`right_polar` cotangents sensitive to gauge choice: (|Δgauge| = $Δgauge)"
return ΔA
end
28 changes: 22 additions & 6 deletions src/pushforwards/polar.jl
Original file line number Diff line number Diff line change
@@ -1,25 +1,41 @@
function left_polar_pushforward!(ΔA, A, WP, ΔWP; kwargs...)
function left_polar_pushforward!(
ΔA, A, WP, ΔWP; rank_atol::Real = default_pullback_rank_atol(WP[2]), kwargs...
)
W, P = WP
ΔW, ΔP = ΔWP
D, U = eigh_full(P)
d = diagview(D)
mul!(ΔP, adjoint(W), ΔA, +1, 0)
K̇ = _sylvester(P, P, adjoint(ΔP) - ΔP)
mul!(ΔW, ΔA, inv(P), +1, 0)
K̇, = _sylvester_psd(U, d, ΔP - adjoint(ΔP), rank_atol)
# ΔA * pinv(P)
ΔAU = ΔA * U
ΔAU .*= transpose(inv_safe.(d, rank_atol))
mul!(ΔW, ΔAU, adjoint(U), +1, 0)
WᴴdAiP = W' * ΔW
mul!(ΔW, W, WᴴdAiP, -1, +1)
ΔW = mul!(ΔW, W, K̇, +1, +1)
ΔP = mul!(ΔP, K̇, P, -1, +1)
ΔP = _remove_null_block!(ΔP, U, d, rank_atol)
return (ΔW, ΔP)
end

function right_polar_pushforward!(ΔA, A, PWᴴ, ΔPWᴴ; kwargs...)
function right_polar_pushforward!(
ΔA, A, PWᴴ, ΔPWᴴ; rank_atol::Real = default_pullback_rank_atol(PWᴴ[1]), kwargs...
)
P, Wᴴ = PWᴴ
ΔP, ΔWᴴ = ΔPWᴴ
D, U = eigh_full(P)
d = diagview(D)
mul!(ΔP, ΔA, adjoint(Wᴴ), +1, 0)
K̇ = _sylvester(P, P, adjoint(ΔP) - ΔP)
mul!(ΔWᴴ, inv(P), ΔA, +1, 0)
K̇, = _sylvester_psd(U, d, ΔP - adjoint(ΔP), rank_atol)
# pinv(P) * ΔA
UᴴΔA = U' * ΔA
UᴴΔA .*= inv_safe.(d, rank_atol)
mul!(ΔWᴴ, U, UᴴΔA, +1, 0)
iPdAW = ΔWᴴ * Wᴴ'
mul!(ΔWᴴ, iPdAW, Wᴴ, -1, +1)
ΔWᴴ = mul!(ΔWᴴ, K̇, Wᴴ, +1, +1)
ΔP = mul!(ΔP, P, K̇, -1, +1)
ΔP = _remove_null_block!(ΔP, U, d, rank_atol)
return (ΔWᴴ, ΔP)
end
50 changes: 49 additions & 1 deletion test/testsuite/ad_utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ project_antihermitian_inplace!(A, alg) = project_antihermitian!(A, A, alg)


enzyme_fdm(T) = eltype(T) <: Union{Float32, ComplexF32} ? EnzymeTestUtils.FiniteDifferences.central_fdm(5, 1, max_range = 1.0e-2) : EnzymeTestUtils.FiniteDifferences.central_fdm(5, 1)

const rank_deficient_fdm = EnzymeTestUtils.FiniteDifferences.central_fdm(5, 1, max_range = 1.0e-4)

"""
eigh_wrapper(f, A, alg)
Expand Down Expand Up @@ -376,6 +376,54 @@ function ad_right_polar_setup(A)
return PWᴴ, ΔPWᴴ
end

"""
left_polar_rank_deficient_wrapper(f, L, R, alg)

Wrapper that calls `W, P = f(L * R, alg)` for a rank-deficient product `L * R` and returns
`W * P` and `P`. Parametrizing the input by its factors restricts FD to rank-preserving directions.
"""
left_polar_rank_deficient_wrapper(f, L, R, alg) = left_polar_gauge_invariant_part(f(L * R, alg))

"""
left_polar!_rank_deficient_wrapper(f!, L, R, alg)

In-place variant of [`left_polar_rank_deficient_wrapper`](@ref).
"""
left_polar!_rank_deficient_wrapper(f!, L, R, alg) =
left_polar_gauge_invariant_part(call_and_zero!(f!, L * R, alg))

left_polar_gauge_invariant_part((W, P)) = (W * P, P)

"""
right_polar_rank_deficient_wrapper(f, L, R, alg)

Wrapper that calls `P, Wᴴ = f(L * R, alg)` for a rank-deficient product `L * R` and returns
`P` and `P * Wᴴ`. Parametrizing the input by its factors restricts FD to rank-preserving directions.
"""
right_polar_rank_deficient_wrapper(f, L, R, alg) = right_polar_gauge_invariant_part(f(L * R, alg))

"""
right_polar!_rank_deficient_wrapper(f!, L, R, alg)

In-place variant of [`right_polar_rank_deficient_wrapper`](@ref).
"""
right_polar!_rank_deficient_wrapper(f!, L, R, alg) =
right_polar_gauge_invariant_part(call_and_zero!(f!, L * R, alg))

right_polar_gauge_invariant_part((P, Wᴴ)) = (P, P * Wᴴ)

"""
polar_P_wrapper(f, A, alg)

Wrapper that returns only the positive factor `P` of the polar decomposition `f(A, alg)`.
"""
polar_P_wrapper(f, A, alg) = (F = f(A, alg); f === left_polar ? F[2] : F[1])

function rank_deficient_factors(T, (m, n))
r = div(min(m, n), 2)
return instantiate_matrix(T, (m, r)), instantiate_matrix(T, (r, n))
end

function ad_left_orth_setup(A)
VC = left_orth(A)
ΔVC = structured_randn!.(similar.(VC))
Expand Down
22 changes: 22 additions & 0 deletions test/testsuite/chainrules.jl
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,17 @@ function test_chainrules_polar(
atol = atol, rtol = rtol, rrule_f = rrule_via_ad, check_inferred = false
)
@test isequal(ΔWP, ΔWP_copy)

Ar = instantiate_rank_deficient_matrix(T, sz)
test_rrule(
config, polar_P_wrapper, left_polar ⊢ NoTangent(), Ar, alg ⊢ NoTangent();
atol = atol, rtol = rtol, rrule_f = rrule_via_ad, check_inferred = false
)
L, R = rank_deficient_factors(T, sz)
test_rrule(
config, left_polar_rank_deficient_wrapper, left_polar ⊢ NoTangent(), L, R, alg ⊢ NoTangent();
atol = atol, rtol = rtol, rrule_f = rrule_via_ad, check_inferred = false
)
end
end
@testset "right_polar" begin
Expand All @@ -708,6 +719,17 @@ function test_chainrules_polar(
atol = atol, rtol = rtol, rrule_f = rrule_via_ad, check_inferred = false
)
@test isequal(ΔPWᴴ, ΔPWᴴ_copy)

Ar = instantiate_rank_deficient_matrix(T, sz)
test_rrule(
config, polar_P_wrapper, right_polar ⊢ NoTangent(), Ar, alg ⊢ NoTangent();
atol = atol, rtol = rtol, rrule_f = rrule_via_ad, check_inferred = false
)
L, R = rank_deficient_factors(T, sz)
test_rrule(
config, right_polar_rank_deficient_wrapper, right_polar ⊢ NoTangent(), L, R, alg ⊢ NoTangent();
atol = atol, rtol = rtol, rrule_f = rrule_via_ad, check_inferred = false
)
end
end
end
Expand Down
26 changes: 26 additions & 0 deletions test/testsuite/enzyme/polar.jl
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,19 @@ function test_enzyme_left_polar(
A = instantiate_matrix(T, sz)
test_forward(left_polar, RT, (A, TA), (alg, Const); atol, rtol)
test_forward(call_and_zero!, RT, (left_polar!, Const), (A, TA), (alg, Const); atol, rtol)

A = instantiate_rank_deficient_matrix(T, sz)
test_reverse(polar_P_wrapper, RT, (left_polar, Const), (A, TA), (alg, Const); atol, rtol, fdm = rank_deficient_fdm)
test_forward(polar_P_wrapper, RT, (left_polar, Const), (A, TA), (alg, Const); atol, rtol, fdm = rank_deficient_fdm)
# TODO: Enzyme can't diff through complex BLAS gemm
if eltype(T) <: Real
L, R = rank_deficient_factors(T, sz)
for wrapper in (left_polar_rank_deficient_wrapper, left_polar!_rank_deficient_wrapper)
f = wrapper === left_polar_rank_deficient_wrapper ? left_polar : left_polar!
test_reverse(wrapper, RT, (f, Const), (L, TA), (R, TA), (alg, Const); atol, rtol)
test_forward(wrapper, RT, (f, Const), (L, TA), (R, TA), (alg, Const); atol, rtol)
end
end
end
end
end
Expand All @@ -57,6 +70,19 @@ function test_enzyme_right_polar(
A = instantiate_matrix(T, sz)
test_forward(right_polar, RT, (A, TA), (alg, Const); atol, rtol)
test_forward(call_and_zero!, RT, (right_polar!, Const), (A, TA), (alg, Const); atol, rtol)

A = instantiate_rank_deficient_matrix(T, sz)
test_reverse(polar_P_wrapper, RT, (right_polar, Const), (A, TA), (alg, Const); atol, rtol, fdm = rank_deficient_fdm)
test_forward(polar_P_wrapper, RT, (right_polar, Const), (A, TA), (alg, Const); atol, rtol, fdm = rank_deficient_fdm)
# TODO: Enzyme can't diff through complex BLAS gemm
if eltype(T) <: Real
L, R = rank_deficient_factors(T, sz)
for wrapper in (right_polar_rank_deficient_wrapper, right_polar!_rank_deficient_wrapper)
f = wrapper === right_polar_rank_deficient_wrapper ? right_polar : right_polar!
test_reverse(wrapper, RT, (f, Const), (L, TA), (R, TA), (alg, Const); atol, rtol)
test_forward(wrapper, RT, (f, Const), (L, TA), (R, TA), (alg, Const); atol, rtol)
end
end
end
end
end
Loading
Loading