RetroZilla/security/nss/lib/ckfw/capi/crsa.c

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "ckcapi.h"
#include "secdert.h"

#define SSL3_SHAMD5_HASH_SIZE  36 /* LEN_MD5 (16) + LEN_SHA1 (20) */

/*
 * ckcapi/crsa.c
 *
 * This file implements the NSSCKMDMechnaism and NSSCKMDCryptoOperation objects
 * for the RSA operation on the CAPI cryptoki module.
 */

/*
 * write a Decimal value to a string
 */

static char *
putDecimalString(char *cstr, unsigned long value)
{
  unsigned long tenpower;
  int first = 1;

  for (tenpower=10000000; tenpower; tenpower /= 10) {
    unsigned char digit = (unsigned char )(value/tenpower);
    value = value % tenpower;

    /* drop leading zeros */
    if (first && (0 == digit)) {
      continue;
    }
    first = 0;
    *cstr++ = digit + '0';
  }

  /* if value was zero, put one of them out */
  if (first) {
    *cstr++ = '0';
  }
  return cstr;
}


/*
 * Create a Capi OID string value from a DER OID
 */
static char *
nss_ckcapi_GetOidString
(
  unsigned char *oidTag,
  unsigned int oidTagSize,
  CK_RV *pError
)
{
  unsigned char *oid;
  char *oidStr;
  char *cstr;
  unsigned long value;
  unsigned int oidSize;

  if (DER_OBJECT_ID != *oidTag) {
    /* wasn't an oid */
    *pError = CKR_DATA_INVALID;
    return NULL;
  }
  oid = nss_ckcapi_DERUnwrap(oidTag, oidTagSize, &oidSize, NULL);

  if (oidSize < 2) {
    *pError = CKR_DATA_INVALID;
    return NULL;
  }

  oidStr = nss_ZNEWARRAY( NULL, char, oidSize*4 );
  if ((char *)NULL == oidStr) {
    *pError = CKR_HOST_MEMORY;
    return NULL;
  }
  cstr = oidStr;
  cstr = putDecimalString(cstr, (*oid) / 40);
  *cstr++ = '.';
  cstr = putDecimalString(cstr, (*oid) % 40);
  oidSize--;

  value = 0;
  while (oidSize--) {
    oid++;
    value = (value << 7) + (*oid & 0x7f);
    if (0 == (*oid & 0x80)) {
      *cstr++ = '.';
      cstr = putDecimalString(cstr, value);
      value = 0;
    }
  }

  *cstr = 0; /* NULL terminate */

  if (value != 0) {
    nss_ZFreeIf(oidStr);
    *pError = CKR_DATA_INVALID;
    return NULL;
  }
  return oidStr;
}


/*
 * PKCS #11 sign for RSA expects to take a fully DER-encoded hash value, 
 * which includes the hash OID. CAPI expects to take a Hash Context. While 
 * CAPI does have the capability of setting a raw hash value, it does not 
 * have the ability to sign an arbitrary value. This function tries to
 * reduce the passed in data into something that CAPI could actually sign.
 */
static CK_RV
ckcapi_GetRawHash
(
  const NSSItem *input,
  NSSItem *hash, 
  ALG_ID *hashAlg
)
{
   unsigned char *current;
   unsigned char *algid;
   unsigned char *oid;
   unsigned char *hashData;
   char *oidStr;
   CK_RV error;
   unsigned int oidSize;
   unsigned int size;
   /*
    * there are 2 types of hashes NSS typically tries to sign, regular
    * RSA signature format (with encoded DER_OIDS), and SSL3 Signed hashes.
    * CAPI knows not to add any oids to SSL3_Signed hashes, so if we have any
    * random hash that is exactly the same size as an SSL3 hash, then we can
    * just pass the data through. CAPI has know way of knowing if the value
    * is really a combined hash or some other arbitrary data, so it's safe to
    * handle this case first.
    */
  if (SSL3_SHAMD5_HASH_SIZE == input->size) {
    hash->data = input->data;
    hash->size = input->size;
    *hashAlg = CALG_SSL3_SHAMD5;
    return CKR_OK;
  }

  current = (unsigned char *)input->data;

  /* make sure we have a sequence tag */
  if ((DER_SEQUENCE|DER_CONSTRUCTED) != *current) {
    return CKR_DATA_INVALID;
  }

  /* parse the input block to get 1) the hash oid, and 2) the raw hash value.
   * unfortunatly CAPI doesn't have a builtin function to do this work, so
   * we go ahead and do it by hand here.
   *
   * format is:
   *  SEQUENCE {
   *     SECQUENCE { // algid
   *       OID {}    // oid
   *       ANY {}    // optional params 
   *     }
   *     OCTECT {}   // hash
   */

  /* unwrap */
  algid = nss_ckcapi_DERUnwrap(current,input->size, &size, NULL);
  
  if (algid+size != current+input->size) {
    /* make sure there is not extra data at the end */
    return CKR_DATA_INVALID;
  }

  if ((DER_SEQUENCE|DER_CONSTRUCTED) != *algid) {
    /* wasn't an algid */
    return CKR_DATA_INVALID;
  }
  oid = nss_ckcapi_DERUnwrap(algid, size, &oidSize, &hashData);

  if (DER_OCTET_STRING != *hashData) {
    /* wasn't a hash */
    return CKR_DATA_INVALID;
  }

  /* get the real hash */
  current = hashData;
  size = size - (hashData-algid);
  hash->data = nss_ckcapi_DERUnwrap(current, size, &hash->size, NULL);

  /* get the real oid as a string. Again, Microsoft does not
   * export anything that does this for us */
  oidStr = nss_ckcapi_GetOidString(oid, oidSize, &error);
  if ((char *)NULL == oidStr ) {
    return error;
  }

  /* look up the hash alg from the oid (fortunately CAPI does to this) */ 
  *hashAlg = CertOIDToAlgId(oidStr);
  nss_ZFreeIf(oidStr);
  if (0 == *hashAlg) {
    return CKR_HOST_MEMORY;
  }

  /* hash looks reasonably consistent, we should be able to sign it now */
  return CKR_OK;
}

/*
 * So everyone else in the worlds stores their bignum data MSB first, but not
 * Microsoft, we need to byte swap everything coming into and out of CAPI.
 */
void
ckcapi_ReverseData(NSSItem *item)
{
  int end = (item->size)-1;
  int middle = (item->size)/2;
  unsigned char *buf = item->data;
  int i;

  for (i=0; i < middle; i++) {
    unsigned char  tmp = buf[i];
    buf[i] = buf[end-i];
    buf[end-i] = tmp;
  }
  return;
}

typedef struct ckcapiInternalCryptoOperationRSAPrivStr 
               ckcapiInternalCryptoOperationRSAPriv;
struct ckcapiInternalCryptoOperationRSAPrivStr
{
  NSSCKMDCryptoOperation mdOperation;
  NSSCKMDMechanism     *mdMechanism;
  ckcapiInternalObject *iKey;
  HCRYPTPROV           hProv;
  DWORD                keySpec;
  HCRYPTKEY            hKey;
  NSSItem	       *buffer;
};

/*
 * ckcapi_mdCryptoOperationRSAPriv_Create
 */
static NSSCKMDCryptoOperation *
ckcapi_mdCryptoOperationRSAPriv_Create
(
  const NSSCKMDCryptoOperation *proto,
  NSSCKMDMechanism *mdMechanism,
  NSSCKMDObject *mdKey,
  CK_RV *pError
)
{
  ckcapiInternalObject *iKey = (ckcapiInternalObject *)mdKey->etc;
  const NSSItem *classItem = nss_ckcapi_FetchAttribute(iKey, CKA_CLASS);
  const NSSItem *keyType = nss_ckcapi_FetchAttribute(iKey, CKA_KEY_TYPE);
  ckcapiInternalCryptoOperationRSAPriv *iOperation;
  CK_RV   error;
  HCRYPTPROV  hProv;
  DWORD       keySpec;
  HCRYPTKEY   hKey;

  /* make sure we have the right objects */
  if (((const NSSItem *)NULL == classItem) ||
      (sizeof(CK_OBJECT_CLASS) != classItem->size) ||
      (CKO_PRIVATE_KEY != *(CK_OBJECT_CLASS *)classItem->data) ||
      ((const NSSItem *)NULL == keyType) ||
      (sizeof(CK_KEY_TYPE) != keyType->size) ||
      (CKK_RSA != *(CK_KEY_TYPE *)keyType->data)) {
    *pError =  CKR_KEY_TYPE_INCONSISTENT;
    return (NSSCKMDCryptoOperation *)NULL;
  }

  error = nss_ckcapi_FetchKeyContainer(iKey, &hProv, &keySpec, &hKey);
  if (error != CKR_OK) {
    *pError = error;
    return (NSSCKMDCryptoOperation *)NULL;
  }

  iOperation = nss_ZNEW(NULL, ckcapiInternalCryptoOperationRSAPriv);
  if ((ckcapiInternalCryptoOperationRSAPriv *)NULL == iOperation) {
    *pError = CKR_HOST_MEMORY;
    return (NSSCKMDCryptoOperation *)NULL;
  }
  iOperation->mdMechanism = mdMechanism;
  iOperation->iKey = iKey;
  iOperation->hProv = hProv;
  iOperation->keySpec = keySpec;
  iOperation->hKey = hKey;

  nsslibc_memcpy(&iOperation->mdOperation, 
                 proto, sizeof(NSSCKMDCryptoOperation));
  iOperation->mdOperation.etc = iOperation;

  return &iOperation->mdOperation;
}

static CK_RV
ckcapi_mdCryptoOperationRSAPriv_Destroy
(
  NSSCKMDCryptoOperation *mdOperation,
  NSSCKFWCryptoOperation *fwOperation,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance
)
{
  ckcapiInternalCryptoOperationRSAPriv *iOperation =
       (ckcapiInternalCryptoOperationRSAPriv *)mdOperation->etc;

  if (iOperation->hKey) {
    CryptDestroyKey(iOperation->hKey);
  }
  if (iOperation->buffer) {
    nssItem_Destroy(iOperation->buffer);
  }
  nss_ZFreeIf(iOperation);
  return CKR_OK;
}

static CK_ULONG
ckcapi_mdCryptoOperationRSA_GetFinalLength
(
  NSSCKMDCryptoOperation *mdOperation,
  NSSCKFWCryptoOperation *fwOperation,
  NSSCKMDSession *mdSession,
  NSSCKFWSession *fwSession,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  CK_RV *pError
)
{
  ckcapiInternalCryptoOperationRSAPriv *iOperation =
       (ckcapiInternalCryptoOperationRSAPriv *)mdOperation->etc;
  const NSSItem *modulus = 
       nss_ckcapi_FetchAttribute(iOperation->iKey, CKA_MODULUS);

  return modulus->size;
}


/*
 * ckcapi_mdCryptoOperationRSADecrypt_GetOperationLength
 * we won't know the length until we actually decrypt the
 * input block. Since we go to all the work to decrypt the
 * the block, we'll save if for when the block is asked for
 */
static CK_ULONG
ckcapi_mdCryptoOperationRSADecrypt_GetOperationLength
(
  NSSCKMDCryptoOperation *mdOperation,
  NSSCKFWCryptoOperation *fwOperation,
  NSSCKMDSession *mdSession,
  NSSCKFWSession *fwSession,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  const NSSItem *input,
  CK_RV *pError
)
{
  ckcapiInternalCryptoOperationRSAPriv *iOperation =
       (ckcapiInternalCryptoOperationRSAPriv *)mdOperation->etc;
  BOOL rc;

  /* Microsoft's Decrypt operation works in place. Since we don't want
   * to trash our input buffer, we make a copy of it */
  iOperation->buffer = nssItem_Duplicate((NSSItem *)input, NULL, NULL);
  if ((NSSItem *) NULL == iOperation->buffer) {
    *pError = CKR_HOST_MEMORY;
    return 0;
  }
  /* Sigh, reverse it */
  ckcapi_ReverseData(iOperation->buffer);
  
  rc = CryptDecrypt(iOperation->hKey, 0, TRUE, 0, 
		    iOperation->buffer->data, &iOperation->buffer->size);
  if (!rc) {
    DWORD msError = GetLastError();
    switch (msError) {
    case NTE_BAD_DATA:
      *pError = CKR_ENCRYPTED_DATA_INVALID;
      break;
    case NTE_FAIL:
    case NTE_BAD_UID:
      *pError = CKR_DEVICE_ERROR;
      break;
    default:
      *pError = CKR_GENERAL_ERROR; 
    }
    return 0;
  }

  return iOperation->buffer->size;
}

/*
 * ckcapi_mdCryptoOperationRSADecrypt_UpdateFinal
 *
 * NOTE: ckcapi_mdCryptoOperationRSADecrypt_GetOperationLength is presumed to 
 * have been called previously.
 */
static CK_RV
ckcapi_mdCryptoOperationRSADecrypt_UpdateFinal
(
  NSSCKMDCryptoOperation *mdOperation,
  NSSCKFWCryptoOperation *fwOperation,
  NSSCKMDSession *mdSession,
  NSSCKFWSession *fwSession,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  const NSSItem *input,
  NSSItem *output
)
{
  ckcapiInternalCryptoOperationRSAPriv *iOperation =
       (ckcapiInternalCryptoOperationRSAPriv *)mdOperation->etc;
  NSSItem *buffer = iOperation->buffer;

  if ((NSSItem *)NULL == buffer) {
    return CKR_GENERAL_ERROR;
  }
  nsslibc_memcpy(output->data, buffer->data, buffer->size);
  output->size = buffer->size;
  return CKR_OK;
}

/*
 * ckcapi_mdCryptoOperationRSASign_UpdateFinal
 *
 */
static CK_RV
ckcapi_mdCryptoOperationRSASign_UpdateFinal
(
  NSSCKMDCryptoOperation *mdOperation,
  NSSCKFWCryptoOperation *fwOperation,
  NSSCKMDSession *mdSession,
  NSSCKFWSession *fwSession,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  const NSSItem *input,
  NSSItem *output
)
{
  ckcapiInternalCryptoOperationRSAPriv *iOperation =
       (ckcapiInternalCryptoOperationRSAPriv *)mdOperation->etc;
  CK_RV error = CKR_OK;
  DWORD msError;
  NSSItem hash;
  HCRYPTHASH hHash = 0;
  ALG_ID  hashAlg;
  DWORD  hashSize;
  DWORD  len; /* temp length value we throw away */
  BOOL   rc;

  /*
   * PKCS #11 sign for RSA expects to take a fully DER-encoded hash value, 
   * which includes the hash OID. CAPI expects to take a Hash Context. While 
   * CAPI does have the capability of setting a raw hash value, it does not 
   * have the ability to sign an arbitrary value. This function tries to
   * reduce the passed in data into something that CAPI could actually sign.
   */
  error = ckcapi_GetRawHash(input, &hash, &hashAlg);
  if (CKR_OK != error) {
    goto loser;
  }

  rc = CryptCreateHash(iOperation->hProv, hashAlg, 0, 0, &hHash);
  if (!rc) {
    goto loser;
  }

  /* make sure the hash lens match before we set it */
  len = sizeof(DWORD);
  rc = CryptGetHashParam(hHash, HP_HASHSIZE, (BYTE *)&hashSize, &len, 0);
  if (!rc) {
    goto loser;
  }

  if (hash.size != hashSize) {
    /* The input must have been bad for this to happen */
    error = CKR_DATA_INVALID;
    goto loser;
  }

  /* we have an explicit hash, set it, note that the length is
   * implicit by the hashAlg used in create */
  rc = CryptSetHashParam(hHash, HP_HASHVAL, hash.data, 0);
  if (!rc) {
    goto loser;
  }

  /* OK, we have the data in a hash structure, sign it! */
  rc = CryptSignHash(hHash, iOperation->keySpec, NULL, 0,
                     output->data, &output->size);
  if (!rc) {
    goto loser;
  }

  /* Don't return a signature that might have been broken because of a cosmic
   * ray, or a broken processor, verify that it is valid... */
  rc = CryptVerifySignature(hHash, output->data, output->size, 
                            iOperation->hKey, NULL, 0);
  if (!rc) {
    goto loser;
  }

  /* OK, Microsoft likes to do things completely differently than anyone
   * else. We need to reverse the data we received here */
  ckcapi_ReverseData(output);
  CryptDestroyHash(hHash);
  return CKR_OK;

loser:
  /* map the microsoft error */
  if (CKR_OK == error) {
    msError = GetLastError();
    switch (msError) {
    case ERROR_NOT_ENOUGH_MEMORY:
      error = CKR_HOST_MEMORY;
      break;
    case NTE_NO_MEMORY:
      error = CKR_DEVICE_MEMORY;
      break;
    case ERROR_MORE_DATA:
      return CKR_BUFFER_TOO_SMALL;
    case ERROR_INVALID_PARAMETER: /* these params were derived from the */
    case ERROR_INVALID_HANDLE:    /* inputs, so if they are bad, the input */ 
    case NTE_BAD_ALGID:           /* data is bad */
    case NTE_BAD_HASH:
      error = CKR_DATA_INVALID;
      break;
    case ERROR_BUSY:
    case NTE_FAIL:
    case NTE_BAD_UID:
      error = CKR_DEVICE_ERROR;
      break;
    default:
      error = CKR_GENERAL_ERROR;
      break;
    }
  }
  if (hHash) {
    CryptDestroyHash(hHash);
  }
  return error;
}
  

NSS_IMPLEMENT_DATA const NSSCKMDCryptoOperation
ckcapi_mdCryptoOperationRSADecrypt_proto = {
  NULL, /* etc */
  ckcapi_mdCryptoOperationRSAPriv_Destroy,
  NULL, /* GetFinalLengh - not needed for one shot Decrypt/Encrypt */
  ckcapi_mdCryptoOperationRSADecrypt_GetOperationLength,
  NULL, /* Final - not needed for one shot operation */
  NULL, /* Update - not needed for one shot operation */
  NULL, /* DigetUpdate - not needed for one shot operation */
  ckcapi_mdCryptoOperationRSADecrypt_UpdateFinal,
  NULL, /* UpdateCombo - not needed for one shot operation */
  NULL, /* DigetKey - not needed for one shot operation */
  (void *)NULL /* null terminator */
};

NSS_IMPLEMENT_DATA const NSSCKMDCryptoOperation
ckcapi_mdCryptoOperationRSASign_proto = {
  NULL, /* etc */
  ckcapi_mdCryptoOperationRSAPriv_Destroy,
  ckcapi_mdCryptoOperationRSA_GetFinalLength,
  NULL, /* GetOperationLengh - not needed for one shot Sign/Verify */
  NULL, /* Final - not needed for one shot operation */
  NULL, /* Update - not needed for one shot operation */
  NULL, /* DigetUpdate - not needed for one shot operation */
  ckcapi_mdCryptoOperationRSASign_UpdateFinal,
  NULL, /* UpdateCombo - not needed for one shot operation */
  NULL, /* DigetKey - not needed for one shot operation */
  (void *)NULL /* null terminator */
};

/********** NSSCKMDMechansim functions ***********************/
/*
 * ckcapi_mdMechanismRSA_Destroy
 */
static void
ckcapi_mdMechanismRSA_Destroy
(
  NSSCKMDMechanism *mdMechanism,
  NSSCKFWMechanism *fwMechanism,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance
)
{
  nss_ZFreeIf(fwMechanism);
}

/*
 * ckcapi_mdMechanismRSA_GetMinKeySize
 */
static CK_ULONG
ckcapi_mdMechanismRSA_GetMinKeySize
(
  NSSCKMDMechanism *mdMechanism,
  NSSCKFWMechanism *fwMechanism,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  CK_RV *pError
)
{
  return 384;
}

/*
 * ckcapi_mdMechanismRSA_GetMaxKeySize
 */
static CK_ULONG
ckcapi_mdMechanismRSA_GetMaxKeySize
(
  NSSCKMDMechanism *mdMechanism,
  NSSCKFWMechanism *fwMechanism,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  CK_RV *pError
)
{
  return 16384;
}

/*
 * ckcapi_mdMechanismRSA_DecryptInit
 */
static NSSCKMDCryptoOperation * 
ckcapi_mdMechanismRSA_DecryptInit
(
  NSSCKMDMechanism *mdMechanism,
  NSSCKFWMechanism *fwMechanism,
  CK_MECHANISM     *pMechanism,
  NSSCKMDSession *mdSession,
  NSSCKFWSession *fwSession,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  NSSCKMDObject *mdKey,
  NSSCKFWObject *fwKey,
  CK_RV *pError
)
{
  return ckcapi_mdCryptoOperationRSAPriv_Create(
		&ckcapi_mdCryptoOperationRSADecrypt_proto,
		mdMechanism, mdKey, pError);
}

/*
 * ckcapi_mdMechanismRSA_SignInit
 */
static NSSCKMDCryptoOperation * 
ckcapi_mdMechanismRSA_SignInit
(
  NSSCKMDMechanism *mdMechanism,
  NSSCKFWMechanism *fwMechanism,
  CK_MECHANISM     *pMechanism,
  NSSCKMDSession *mdSession,
  NSSCKFWSession *fwSession,
  NSSCKMDToken *mdToken,
  NSSCKFWToken *fwToken,
  NSSCKMDInstance *mdInstance,
  NSSCKFWInstance *fwInstance,
  NSSCKMDObject *mdKey,
  NSSCKFWObject *fwKey,
  CK_RV *pError
)
{
  return ckcapi_mdCryptoOperationRSAPriv_Create(
		&ckcapi_mdCryptoOperationRSASign_proto,
		mdMechanism, mdKey, pError);
}


NSS_IMPLEMENT_DATA const NSSCKMDMechanism
nss_ckcapi_mdMechanismRSA = {
  (void *)NULL, /* etc */
  ckcapi_mdMechanismRSA_Destroy,
  ckcapi_mdMechanismRSA_GetMinKeySize,
  ckcapi_mdMechanismRSA_GetMaxKeySize,
  NULL, /* GetInHardware - default false */
  NULL, /* EncryptInit - default errs */
  ckcapi_mdMechanismRSA_DecryptInit,
  NULL, /* DigestInit - default errs*/
  ckcapi_mdMechanismRSA_SignInit,
  NULL, /* VerifyInit - default errs */
  ckcapi_mdMechanismRSA_SignInit,  /* SignRecoverInit */
  NULL, /* VerifyRecoverInit - default errs */
  NULL, /* GenerateKey - default errs */
  NULL, /* GenerateKeyPair - default errs */
  NULL, /* GetWrapKeyLength - default errs */
  NULL, /* WrapKey - default errs */
  NULL, /* UnwrapKey - default errs */
  NULL, /* DeriveKey - default errs */
  (void *)NULL /* null terminator */
};