Programmatically verify certificate chain using OpenSSL API

You can use the normal validation routines (see How do you verify a public key was issued by your private CA?), like the -verify function in OpenSSL does. You need to create a lookup method (X509_LOOKUP_METHOD) like X509_LOOKUP_file(), but which works with a character string instead of a filename. The code for X509_LOOKUP_buffer() is as follows.

Header file by_buffer.h:

/* File:   by_buffer.h */

#ifndef BY_BUFFER_H
#define    BY_BUFFER_H

#include <openssl/x509.h>

#ifdef    __cplusplus
extern "C" {
#endif
#define X509_L_BUF_LOAD    1
#define X509_LOOKUP_load_buf(x,name,type) \
        X509_LOOKUP_ctrl((x),X509_L_BUF_LOAD,(name),(long)(type),NULL)
X509_LOOKUP_METHOD *X509_LOOKUP_buffer(void);

#ifdef    __cplusplus
}
#endif

#endif    /* BY_BUFFER_H */

The c program by_buffer.c:

/* by_buffer.c - copied and modified from crypto/x509/by_file.c */
/* Copyright (C) - should be the same as for OpenSSL
*/
#include "by_buffer.h"

#include <stdio.h>
#include <time.h>
#include <errno.h>

#include "../crypto/cryptlib.h"
#include <openssl/lhash.h>
#include <openssl/buffer.h>
#include <openssl/pem.h>
#include <openssl/err.h>

static int by_buffer_ctrl(X509_LOOKUP *ctx, int cmd, const char *argc,
    long argl, char **ret);
X509_LOOKUP_METHOD x509_buffer_lookup=
    {
    "Load buffer into cache",
    NULL,        /* new */
    NULL,        /* free */
    NULL,         /* init */
    NULL,        /* shutdown */
    by_buffer_ctrl,    /* ctrl */
    NULL,        /* get_by_subject */
    NULL,        /* get_by_issuer_serial */
    NULL,        /* get_by_fingerprint */
    NULL,        /* get_by_alias */
    };

X509_LOOKUP_METHOD *X509_LOOKUP_buffer(void)
    {
    return(&x509_buffer_lookup);
    }

static int by_buffer_ctrl(X509_LOOKUP *ctx, int cmd, const char *argp, long argl,
         char **ret)
    {
    int ok=0;
    char *certBuf;

    switch (cmd)
        {
    case X509_L_BUF_LOAD:
        if (argl == X509_FILETYPE_DEFAULT)
            {
            X509err(X509_F_BY_FILE_CTRL,X509_R_LOADING_DEFAULTS);
            }
        else
            {
            if(argl == X509_FILETYPE_PEM)
                ok = (X509_load_cert_crl_buf(ctx,argp,
                    X509_FILETYPE_PEM) != 0);
            else
                ok = (X509_load_cert_buf(ctx,argp,(int)argl) != 0);
            }
        break;
        }
    return(ok);
    }

int X509_load_cert_buf(X509_LOOKUP *ctx, const char *certBuf, int type)
    {
    int ret=0;
    BIO *in=NULL;
    int i,count=0;
    X509 *x=NULL;

    if (certBuf == NULL) return(1);
        in=BIO_new(BIO_s_mem());
        if(in==NULL) goto err;

    if (type == X509_FILETYPE_PEM)
        {
        for (;;)
            {
            x=PEM_read_bio_X509_AUX(in,NULL,NULL,NULL);
            if (x == NULL)
                {
                if ((ERR_GET_REASON(ERR_peek_last_error()) ==
                    PEM_R_NO_START_LINE) && (count > 0))
                    {
                    ERR_clear_error();
                    break;
                    }
                else
                    {
                    X509err(X509_F_X509_LOAD_CERT_FILE,
                        ERR_R_PEM_LIB);
                    goto err;
                    }
                }
            i=X509_STORE_add_cert(ctx->store_ctx,x);
            if (!i) goto err;
            count++;
            X509_free(x);
            x=NULL;
            }
        ret=count;
        }
    else if (type == X509_FILETYPE_ASN1)
        {
        x=d2i_X509_bio(in,NULL);
        if (x == NULL)
            {
            X509err(X509_F_X509_LOAD_CERT_FILE,ERR_R_ASN1_LIB);
            goto err;
            }
        i=X509_STORE_add_cert(ctx->store_ctx,x);
        if (!i) goto err;
        ret=i;
        }
    else
        {
        X509err(X509_F_X509_LOAD_CERT_FILE,X509_R_BAD_X509_FILETYPE);
        goto err;
        }
err:
    if (x != NULL) X509_free(x);
    if (in != NULL) BIO_free(in);
    return(ret);
    }

int X509_load_crl_buf(X509_LOOKUP *ctx, const char *certBuf, int type)
    {
    int ret=0;
    BIO *in=NULL;
    int i,count=0;
    X509_CRL *x=NULL;

    if (certBuf == NULL) return(1);
    //in=BIO_new(BIO_s_file_internal());
        in=BIO_new(BIO_s_mem());

        if(in==NULL) goto err;

    if (type == X509_FILETYPE_PEM)
        {
        for (;;)
            {
            x=PEM_read_bio_X509_CRL(in,NULL,NULL,NULL);
            if (x == NULL)
                {
                if ((ERR_GET_REASON(ERR_peek_last_error()) ==
                    PEM_R_NO_START_LINE) && (count > 0))
                    {
                    ERR_clear_error();
                    break;
                    }
                else
                    {
                    X509err(X509_F_X509_LOAD_CRL_FILE,
                        ERR_R_PEM_LIB);
                    goto err;
                    }
                }
            i=X509_STORE_add_crl(ctx->store_ctx,x);
            if (!i) goto err;
            count++;
            X509_CRL_free(x);
            x=NULL;
            }
        ret=count;
        }
    else if (type == X509_FILETYPE_ASN1)
        {
        x=d2i_X509_CRL_bio(in,NULL);
        if (x == NULL)
            {
            X509err(X509_F_X509_LOAD_CRL_FILE,ERR_R_ASN1_LIB);
            goto err;
            }
        i=X509_STORE_add_crl(ctx->store_ctx,x);
        if (!i) goto err;
        ret=i;
        }
    else
        {
        X509err(X509_F_X509_LOAD_CRL_FILE,X509_R_BAD_X509_FILETYPE);
        goto err;
        }
err:
    if (x != NULL) X509_CRL_free(x);
    if (in != NULL) BIO_free(in);
    return(ret);
    }

int X509_load_cert_crl_buf(X509_LOOKUP *ctx, const char *certBuf, int type)
{
    STACK_OF(X509_INFO) *inf;
    X509_INFO *itmp;
    BIO *in;
    int i, count = 0;
    if(type != X509_FILETYPE_PEM)
        return X509_load_cert_buf(ctx, certBuf, type);
        in = BIO_new(BIO_s_mem());
    if(!in) {
        X509err(X509_F_X509_LOAD_CERT_CRL_FILE,ERR_R_SYS_LIB);
        return 0;
    }
        BIO_write(in, certBuf, strlen(certBuf));
    inf = PEM_X509_INFO_read_bio(in, NULL, NULL, NULL);
    BIO_free(in);
    if(!inf) {
        X509err(X509_F_X509_LOAD_CERT_CRL_FILE,ERR_R_PEM_LIB);
        return 0;
    }
    for(i = 0; i < sk_X509_INFO_num(inf); i++) {
        itmp = sk_X509_INFO_value(inf, i);
        if(itmp->x509) {
            X509_STORE_add_cert(ctx->store_ctx, itmp->x509);
            count++;
        }
        if(itmp->crl) {
            X509_STORE_add_crl(ctx->store_ctx, itmp->crl);
            count++;
        }
    }
    sk_X509_INFO_pop_free(inf, X509_INFO_free);
    return count;
}

Routine in C++ which calls the above routines:

#include "by_buffer.h"
static int check(X509_STORE *ctx, const char *certBuf);
static X509 *load_cert(const char *certBuf);

int validateKey(const char *rsaKeyCA, const char *rsaCertificate) {
    int ret=0;
    X509_STORE *cert_ctx=NULL;
    X509_LOOKUP *lookup=NULL;

    cert_ctx=X509_STORE_new();
    if (cert_ctx == NULL) goto end;

    OpenSSL_add_all_algorithms();

    lookup=X509_STORE_add_lookup(cert_ctx,X509_LOOKUP_buffer());
    if (lookup == NULL)
        goto end;

    if(!X509_LOOKUP_load_buf(lookup,rsaKeyCA,X509_FILETYPE_PEM))
        goto end;

    lookup=X509_STORE_add_lookup(cert_ctx,X509_LOOKUP_hash_dir());
    if (lookup == NULL)
        goto end;

    X509_LOOKUP_add_dir(lookup,NULL,X509_FILETYPE_DEFAULT);

    ret = check(cert_ctx, rsaCertificate);
end:
    if (cert_ctx != NULL) X509_STORE_free(cert_ctx);

    return ret;
}

static X509 *load_cert(const char *certBuf)
{
    X509 *x=NULL;
    BIO *cert;

    if ((cert=BIO_new(BIO_s_mem())) == NULL)
        goto end;

    BIO_write(cert, certBuf, strlen(certBuf));

    x=PEM_read_bio_X509_AUX(cert,NULL, NULL, NULL);
end:
    if (cert != NULL) BIO_free(cert);
    return(x);
}

static int check(X509_STORE *ctx, const char *certBuf)
{
    X509 *x=NULL;
    int i=0,ret=0;
    X509_STORE_CTX *csc;

    x = load_cert(certBuf);
    if (x == NULL)
        goto end;

    csc = X509_STORE_CTX_new();
    if (csc == NULL)
        goto end;
    X509_STORE_set_flags(ctx, 0);
    if(!X509_STORE_CTX_init(csc,ctx,x,0))
        goto end;
    ////// See crypto/asn1/t_x509.c for ideas on how to access and print the values
    //printf("X.509 name: %s\n", x->name);
    i=X509_verify_cert(csc);
    X509_STORE_CTX_free(csc);

    ret=0;
end:
    ret = (i > 0);
    if (x != NULL)
        X509_free(x);

    return(ret);
}