@@ -14,6 +14,7 @@ use openssl::{
1414
1515use crate :: {
1616 CryptoError , crypto:: rsa:: default_cryptographic_parameters, error:: result:: CryptoResult ,
17+ openssl:: hashing_algorithm_to_openssl_ref,
1718} ;
1819
1920/// Attempt to sign a digest using an RSA private key and the provided algorithm name.
@@ -84,43 +85,63 @@ pub fn sign_rsa_with_pkey(request: &Sign, private_key: &PKey<Private>) -> Crypto
8485 let ( _algorithm, _padding, default_hash, digital_signature_algorithm) =
8586 default_cryptographic_parameters ( request. cryptographic_parameters . as_ref ( ) ) ;
8687
87- // Determine effective message digest
88- let digest = if let Some ( cp) = request. cryptographic_parameters . as_ref ( ) {
89- if let Some ( h) = & cp. hashing_algorithm {
90- match h {
91- KmipHash :: SHA1 => MessageDigest :: sha1 ( ) ,
92- KmipHash :: SHA256 => MessageDigest :: sha256 ( ) ,
93- KmipHash :: SHA384 => MessageDigest :: sha384 ( ) ,
94- KmipHash :: SHA512 => MessageDigest :: sha512 ( ) ,
95- KmipHash :: SHA3256 => MessageDigest :: sha3_256 ( ) ,
96- KmipHash :: SHA3384 => MessageDigest :: sha3_384 ( ) ,
97- KmipHash :: SHA3512 => MessageDigest :: sha3_512 ( ) ,
98- _ => {
99- return Err ( CryptoError :: Default (
100- "sign_rsa_with_pkey: hashing algorithm not supported" . to_owned ( ) ,
101- ) ) ;
102- }
103- }
88+ // Determine effective hashing algorithm (KMIP) first, then map to OpenSSL.
89+ let mut effective_hash: KmipHash = if let Some ( cp) = request. cryptographic_parameters . as_ref ( ) {
90+ if let Some ( h) = cp. hashing_algorithm {
91+ h
10492 } else {
10593 match digital_signature_algorithm {
10694 DigitalSignatureAlgorithm :: RSASSAPSS
107- | DigitalSignatureAlgorithm :: SHA256WithRSAEncryption => MessageDigest :: sha256 ( ) ,
108- DigitalSignatureAlgorithm :: SHA384WithRSAEncryption => MessageDigest :: sha384 ( ) ,
109- DigitalSignatureAlgorithm :: SHA512WithRSAEncryption => MessageDigest :: sha512 ( ) ,
110- DigitalSignatureAlgorithm :: SHA3256WithRSAEncryption => MessageDigest :: sha3_256 ( ) ,
111- DigitalSignatureAlgorithm :: SHA3384WithRSAEncryption => MessageDigest :: sha3_384 ( ) ,
112- DigitalSignatureAlgorithm :: SHA3512WithRSAEncryption => MessageDigest :: sha3_512 ( ) ,
113- _ => {
95+ | DigitalSignatureAlgorithm :: SHA256WithRSAEncryption => KmipHash :: SHA256 ,
96+ DigitalSignatureAlgorithm :: SHA384WithRSAEncryption => KmipHash :: SHA384 ,
97+ DigitalSignatureAlgorithm :: SHA512WithRSAEncryption => KmipHash :: SHA512 ,
98+ DigitalSignatureAlgorithm :: SHA3256WithRSAEncryption => KmipHash :: SHA3256 ,
99+ DigitalSignatureAlgorithm :: SHA3384WithRSAEncryption => KmipHash :: SHA3384 ,
100+ DigitalSignatureAlgorithm :: SHA3512WithRSAEncryption => KmipHash :: SHA3512 ,
101+ other => {
114102 return Err ( CryptoError :: Default ( format ! (
115- "sign_rsa_with_pkey: not supported: {digital_signature_algorithm :?}"
103+ "sign_rsa_with_pkey: not supported: {other :?}"
116104 ) ) ) ;
117105 }
118106 }
119107 }
108+ } else if let Some ( digested_data) = & request. digested_data {
109+ // When no cryptographic parameters are provided but we have digested data,
110+ // infer the digest algorithm from the size of the digest
111+ match digested_data. len ( ) {
112+ 20 => KmipHash :: SHA1 ,
113+ 32 => KmipHash :: SHA256 ,
114+ 48 => KmipHash :: SHA384 ,
115+ 64 => KmipHash :: SHA512 ,
116+ _ => default_hash,
117+ }
120118 } else {
121- map_kmip_hash_to_openssl ( default_hash)
119+ default_hash
122120 } ;
123121
122+ // If the caller provided pre-digested data, prefer inferring the hash from the digest length.
123+ // This avoids accidentally selecting SHA-1 (disallowed in FIPS) when the digest is clearly
124+ // SHA-256/384/512.
125+ if let Some ( digested_data) = & request. digested_data {
126+ effective_hash = match digested_data. len ( ) {
127+ 20 => KmipHash :: SHA1 ,
128+ 32 => KmipHash :: SHA256 ,
129+ 48 => KmipHash :: SHA384 ,
130+ 64 => KmipHash :: SHA512 ,
131+ _ => effective_hash,
132+ } ;
133+ }
134+
135+ // OpenSSL FIPS provider forbids SHA-1 for RSA signing.
136+ #[ cfg( not( feature = "non-fips" ) ) ]
137+ if effective_hash == KmipHash :: SHA1 {
138+ return Err ( CryptoError :: Default (
139+ "RSA signing with SHA-1 is not supported in FIPS mode" . to_owned ( ) ,
140+ ) ) ;
141+ }
142+
143+ let digest = map_kmip_hash_to_openssl ( effective_hash) ;
144+
124145 // RSASSA-PSS: pre-hash path when digested_data provided
125146 if digital_signature_algorithm == DigitalSignatureAlgorithm :: RSASSAPSS
126147 && request. digested_data . is_some ( )
@@ -131,24 +152,30 @@ pub fn sign_rsa_with_pkey(request: &Sign, private_key: &PKey<Private>) -> Crypto
131152 }
132153 let mut ctx = PkeyCtx :: new ( private_key) ?;
133154 ctx. sign_init ( ) ?;
134- ctx. set_rsa_padding ( Padding :: PKCS1_PSS ) ?;
155+ let mgf1_hash = request
156+ . cryptographic_parameters
157+ . as_ref ( )
158+ . and_then ( |cp| cp. mask_generator_hashing_algorithm )
159+ . unwrap_or ( effective_hash) ;
135160
136- if let Some ( cp) = request. cryptographic_parameters . as_ref ( ) {
137- if let Some ( h) = cp. mask_generator_hashing_algorithm {
138- let mgf1 = map_kmip_hash_to_openssl ( h) ;
139- #[ allow( unsafe_code) ]
140- ctx. set_rsa_mgf1_md ( unsafe { & * ( mgf1. as_ptr ( ) . cast :: < openssl:: md:: MdRef > ( ) ) } ) ?;
141- } else {
142- #[ allow( unsafe_code) ]
143- ctx. set_rsa_mgf1_md ( unsafe { & * ( digest. as_ptr ( ) . cast :: < openssl:: md:: MdRef > ( ) ) } ) ?;
144- }
161+ #[ cfg( not( feature = "non-fips" ) ) ]
162+ let mgf1_hash = if mgf1_hash == KmipHash :: SHA1 {
163+ effective_hash
145164 } else {
146- #[ allow( unsafe_code) ]
147- ctx. set_rsa_mgf1_md ( unsafe { & * ( digest. as_ptr ( ) . cast :: < openssl:: md:: MdRef > ( ) ) } ) ?;
148- }
149- // Tell OpenSSL what the hash type is
150- #[ allow( unsafe_code) ]
151- ctx. set_signature_md ( unsafe { & * ( digest. as_ptr ( ) . cast :: < openssl:: md:: MdRef > ( ) ) } ) ?;
165+ mgf1_hash
166+ } ;
167+
168+ // OpenSSL FIPS provider forbids SHA-1 for RSA PSS MGF1.
169+ // KMIP says the default MGF1 hash is SHA-1 when omitted, so we must
170+ // override that default to match the signature hash (typically SHA-256)
171+ // for FIPS compatibility.
172+ // Set the signature digest first so OpenSSL doesn't initialize RSA-PSS
173+ // with SHA-1 defaults (disallowed in FIPS).
174+ ctx. set_signature_md ( hashing_algorithm_to_openssl_ref ( effective_hash) ?) ?;
175+ // Then select PSS padding; the digest is already configured.
176+ ctx. set_rsa_padding ( Padding :: PKCS1_PSS ) ?;
177+ // MGF1 digest is a PSS-only parameter, so set it after PSS is selected.
178+ ctx. set_rsa_mgf1_md ( hashing_algorithm_to_openssl_ref ( mgf1_hash) ?) ?;
152179
153180 let salt_len = request
154181 . cryptographic_parameters
@@ -162,19 +189,77 @@ pub fn sign_rsa_with_pkey(request: &Sign, private_key: &PKey<Private>) -> Crypto
162189 } ) ?;
163190 buffer. extend_from_slice ( digested_data) ;
164191 // First call: Pass None to get the required buffer size
165- let required_len = ctx. sign ( & buffer, None ) ?;
192+ let required_len = ctx. sign ( & buffer, None ) . map_err ( |e| {
193+ CryptoError :: Default ( format ! (
194+ "rsa pss prehash sign init failed (hash={effective_hash:?}, mgf1={mgf1_hash:?}, payload_len={}): {e}" ,
195+ buffer. len( )
196+ ) )
197+ } ) ?;
166198 // Second call: Pass a buffer of the correct size
167199 let mut signature = vec ! [ 0_u8 ; required_len] ;
168- ctx. sign ( & buffer, Some ( & mut signature) ) ?;
200+ ctx. sign ( & buffer, Some ( & mut signature) ) . map_err ( |e| {
201+ CryptoError :: Default ( format ! (
202+ "rsa pss prehash sign failed (hash={effective_hash:?}, mgf1={mgf1_hash:?}, payload_len={}): {e}" ,
203+ buffer. len( )
204+ ) )
205+ } ) ?;
206+ return Ok ( signature) ;
207+ }
208+
209+ // PKCS#1 v1.5: pre-hash path when digested_data provided
210+ // Use PkeyCtx for FIPS compatibility with pre-digested data
211+ if request. digested_data . is_some ( ) {
212+ let digested_data = request
213+ . digested_data
214+ . as_ref ( )
215+ . ok_or_else ( || CryptoError :: ObjectNotFound ( "Missing digested data" . to_owned ( ) ) ) ?;
216+
217+ let mut buffer = Vec :: new ( ) ;
218+ if let Some ( corr) = & request. correlation_value {
219+ buffer. extend_from_slice ( corr) ;
220+ }
221+ buffer. extend_from_slice ( digested_data) ;
222+
223+ let mut ctx = PkeyCtx :: new ( private_key) ?;
224+ ctx. sign_init ( ) ?;
225+ ctx. set_signature_md ( hashing_algorithm_to_openssl_ref ( effective_hash) ?) ?;
226+ // Tell OpenSSL what the hash type is for the pre-digested data.
227+ // Set it before padding so OpenSSL doesn't use SHA-1 defaults.
228+ ctx. set_rsa_padding ( Padding :: PKCS1 ) ?;
229+
230+ // First call: Pass None to get the required buffer size
231+ let required_len = ctx. sign ( & buffer, None ) . map_err ( |e| {
232+ CryptoError :: Default ( format ! (
233+ "rsa pkcs1 prehash sign init failed (hash={effective_hash:?}, payload_len={}): {e}" ,
234+ buffer. len( )
235+ ) )
236+ } ) ?;
237+ // Second call: Pass a buffer of the correct size
238+ let mut signature = vec ! [ 0_u8 ; required_len] ;
239+ ctx. sign ( & buffer, Some ( & mut signature) ) . map_err ( |e| {
240+ CryptoError :: Default ( format ! (
241+ "rsa pkcs1 prehash sign failed (hash={effective_hash:?}, payload_len={}): {e}" ,
242+ buffer. len( )
243+ ) )
244+ } ) ?;
169245 return Ok ( signature) ;
170246 }
171247
248+ // Standard path for non-digested data
172249 let mut signer = Signer :: new ( digest, private_key) ?;
173250 if DigitalSignatureAlgorithm :: RSASSAPSS == digital_signature_algorithm {
174251 signer. set_rsa_padding ( Padding :: PKCS1_PSS ) ?;
175252 if let Some ( cp) = request. cryptographic_parameters . as_ref ( ) {
176253 if let Some ( h) = cp. mask_generator_hashing_algorithm {
177- let mgf1 = map_kmip_hash_to_openssl ( h) ;
254+ let mgf1_hash = h;
255+
256+ #[ cfg( not( feature = "non-fips" ) ) ]
257+ let mgf1_hash = if mgf1_hash == KmipHash :: SHA1 {
258+ effective_hash
259+ } else {
260+ mgf1_hash
261+ } ;
262+ let mgf1 = map_kmip_hash_to_openssl ( mgf1_hash) ;
178263 signer. set_rsa_mgf1_md ( mgf1) ?;
179264 } else {
180265 signer. set_rsa_mgf1_md ( digest) ?;
0 commit comments