ITS#5696, more for MozNSS. Not usable by default.

include/portable.hin

@@ -397,6 +397,9 @@
 /* define if you have OpenSSL */
 #undef HAVE_OPENSSL
 
+/* define if you have MozNSS */
+#undef HAVE_MOZNSS
+
 /* Define to 1 if you have the <openssl/bn.h> header file. */
 #undef HAVE_OPENSSL_BN_H
 

libraries/liblutil/passwd.c

@@ -34,7 +34,25 @@
 #include <ac/unistd.h>
 
 #if defined(SLAPD_LMHASH)
+#if defined(HAVE_OPENSSL)
 #	include <openssl/des.h>
+
+
+typedef des_cblock des_key;
+typedef des_cblock des_data_block;
+typedef des_key_schedule des_context;
+#define des_failed(encrypted) 0
+#define des_finish(key, schedule) 
+
+#elif defined(HAVE_MOZNSS)
+#	include <pk11pub.h>
+typedef PK11SymKey *des_key;
+typedef unsigned char des_data_block[8];
+typedef PK11Context *des_context[1];
+#define DES_ENCRYPT CKA_ENCRYPT
+
+#endif
+
 #endif /* SLAPD_LMHASH */
 
 #include <ac/param.h>
@@ -632,6 +650,106 @@ static int chk_md5(
 }
 
 #ifdef SLAPD_LMHASH
+
+#if defined(HAVE_OPENSSL)
+
+/*
+ * abstract away setting the parity.
+ */
+static void
+des_set_key_and_parity( des_key *key, unsigned char *keyData)
+{
+    memcpy(key, keyData, 8);
+    des_set_odd_parity( key );
+}
+
+
+#elif defined(HAVE_MOZNSS)
+
+/*
+ * implement MozNSS wrappers for the openSSL calls 
+ */
+static void
+des_set_key_and_parity( des_key *key, unsigned char *keyData)
+{
+    SECItem keyDataItem;
+    PK11SlotInfo *slot;
+    *key = NULL;
+
+    keyDataItem.data = keyData;
+    keyDataItem.len = 8;
+
+    slot = PK11_GetBestSlot(CKM_DES_ECB, NULL);
+    if (slot == NULL) {
+	return;
+    }
+
+    /* NOTE: this will not work in FIPS mode. In order to make lmhash
+     * work in fips mode we need to define a LMHASH pbe mechanism and
+     * do the fulll key derivation inside the token */
+    *key = PK11_ImportSymKey(slot, CKM_DES_ECB, PK11_OriginGenerated, 
+		CKA_ENCRYPT, &keyDataItem, NULL);
+}
+
+static void
+des_set_key_unchecked( des_key *key, des_context ctxt )
+{
+    ctxt[0] = NULL;
+
+    /* handle error conditions from previous call */
+    if (!*key) {
+	return;
+    }
+
+    ctxt[0] = PK11_CreateContextBySymKey(CKM_DES_ECB, CKA_ENCRYPT, *key, NULL);
+}
+
+static void
+des_ecb_encrypt( des_data_block *plain, des_data_block *encrypted, 
+			des_context ctxt, int op)
+{
+    SECStatus rv;
+    int size;
+
+    if (ctxt[0] == NULL) {
+	/* need to fail here...  */
+	memset(encrypted, 0, sizeof(des_data_block));
+	return;
+    }
+    rv = PK11_CipherOp(ctxt[0], (unsigned char *)&encrypted[0], 
+			&size, sizeof(des_data_block),
+			(unsigned char *)&plain[0], sizeof(des_data_block));
+    if (rv != SECSuccess) {
+	/* signal failure */
+	memset(encrypted, 0, sizeof(des_data_block));
+	return;
+    }
+    return;
+}
+
+static int
+des_failed(des_data_block *encrypted)
+{
+   static const des_data_block zero = { 0 };
+   return memcmp(encrypted, zero, sizeof(zero)) == 0;
+}
+
+static void
+des_finish(des_key *key, des_context ctxt)
+{
+     if (*key) {
+	PK11_FreeSymKey(*key);
+	*key = NULL;
+     }
+     if (ctxt[0]) {
+	PK11_Finalize(ctxt[0]);
+	PK11_DestroyContext(ctxt[0], PR_TRUE);
+	ctxt[0] = NULL;
+     }
+}
+
+#endif
+
 /* pseudocode from RFC2433
  * A.2 LmPasswordHash()
  * 
@@ -692,10 +810,10 @@ static int chk_md5(
 
 static void lmPasswd_to_key(
 	const char *lmPasswd,
-	des_cblock *key)
+	des_key *key)
 {
 	const unsigned char *lpw = (const unsigned char *) lmPasswd;
-	unsigned char *k = (unsigned char *) key;
+	unsigned char k[8];
 
 	/* make room for parity bits */
 	k[0] = lpw[0];
@@ -707,7 +825,7 @@ static void lmPasswd_to_key(
 	k[6] = ((lpw[5] & 0x3F) << 2) | (lpw[6] >> 6);
 	k[7] = ((lpw[6] & 0x7F) << 1);
 		
-	des_set_odd_parity( key );
+	des_set_key_and_parity( key, k );
 }	
 
 static int chk_lanman(
@@ -718,10 +836,10 @@ static int chk_lanman(
 {
 	ber_len_t i;
 	char UcasePassword[15];
-	des_cblock key;
-	des_key_schedule schedule;
-	des_cblock StdText = "KGS!@#$%";
-	des_cblock PasswordHash1, PasswordHash2;
+	des_key key;
+	des_context schedule;
+	des_data_block StdText = "KGS!@#$%";
+	des_data_block PasswordHash1, PasswordHash2;
 	char PasswordHash[33], storedPasswordHash[33];
 	
 	for( i=0; i<cred->bv_len; i++) {
@@ -741,10 +859,19 @@ static int chk_lanman(
 	lmPasswd_to_key( UcasePassword, &key );
 	des_set_key_unchecked( &key, schedule );
 	des_ecb_encrypt( &StdText, &PasswordHash1, schedule , DES_ENCRYPT );
+
+	if (des_failed(&PasswordHash1)) {
+	    return LUTIL_PASSWD_ERR;
+	}
 	
 	lmPasswd_to_key( &UcasePassword[7], &key );
 	des_set_key_unchecked( &key, schedule );
 	des_ecb_encrypt( &StdText, &PasswordHash2, schedule , DES_ENCRYPT );
+	if (des_failed(&PasswordHash2)) {
+	    return LUTIL_PASSWD_ERR;
+	}
+
+	des_finish( &key, schedule );
 	
 	sprintf( PasswordHash, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", 
 		PasswordHash1[0],PasswordHash1[1],PasswordHash1[2],PasswordHash1[3],
@@ -1005,10 +1132,10 @@ static int hash_lanman(
 
 	ber_len_t i;
 	char UcasePassword[15];
-	des_cblock key;
-	des_key_schedule schedule;
-	des_cblock StdText = "KGS!@#$%";
-	des_cblock PasswordHash1, PasswordHash2;
+	des_key key;
+	des_context schedule;
+	des_data_block StdText = "KGS!@#$%";
+	des_data_block PasswordHash1, PasswordHash2;
 	char PasswordHash[33];
 	
 	for( i=0; i<passwd->bv_len; i++) {