remove unnecessary and error-prone allocation (+ add openssl 1.1 changes)

[matu3ba]

trafficstars

The structure has only 5 ints, which is not that big that it does not fit on the stack. This simplifies file_encrypt_decrypt to this function:

#include <openssl/err.h>
#include <openssl/evp.h>

#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define ERR_EVP_CIPHER_INIT -1
#define ERR_EVP_CIPHER_UPDATE -2
#define ERR_EVP_CIPHER_FINAL -3
#define ERR_EVP_CTX_NEW -4

#define AES_256_KEY_SIZE 32
#define AES_BLOCK_SIZE 16
#define BUFSIZE 1024

typedef struct _cipher_params_t{
    unsigned char *key;
    unsigned char *iv;
    unsigned int encrypt;
    const EVP_CIPHER *cipher_type;
}cipher_params_t;

// adjusted from https://github.com/kulkarniamit/openssl-evp-demo
// 1e9f2c2afe267d98e7530a16028d798df09d49a2 ./openssl_evp_demo.c
int file_encrypt_decrypt(cipher_params_t *params, FILE *ifp, FILE *ofp)
{
    // enough space in output buffer for additional block
    int cipher_block_size = EVP_CIPHER_block_size(params->cipher_type);
    unsigned char in_buf[BUFSIZE], out_buf[BUFSIZE + cipher_block_size];

    int num_bytes_read, out_len;
    EVP_CIPHER_CTX *ctx;

    ctx = EVP_CIPHER_CTX_new(); // potentially allocates
    if(ctx == NULL){
        fprintf(stderr, "ERROR: EVP_CIPHER_CTX_new failed. OpenSSL error: %s\n", ERR_error_string(ERR_get_error(), NULL));
        return ERR_EVP_CTX_NEW;
    }

    // check lengths
    if(!EVP_CipherInit_ex(ctx, params->cipher_type, NULL, NULL, NULL, params->encrypt)){
        EVP_CIPHER_CTX_free(ctx);
        return ERR_EVP_CIPHER_INIT;
    }

    OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) == AES_256_KEY_SIZE);
    OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == AES_BLOCK_SIZE);

    // set key and iv
    if(!EVP_CipherInit_ex(ctx, NULL, NULL, params->key, params->iv, params->encrypt)){
        EVP_CIPHER_CTX_free(ctx);
        return ERR_EVP_CIPHER_INIT;
    }

    while(1){
        // read in data in blocks and update cipher on every read
        num_bytes_read = fread(in_buf, sizeof(unsigned char), BUFSIZE, ifp);
        if (ferror(ifp)){
            EVP_CIPHER_CTX_free(ctx);
            return errno;
        }
        if(!EVP_CipherUpdate(ctx, out_buf, &out_len, in_buf, num_bytes_read)){
            EVP_CIPHER_CTX_free(ctx);
            return ERR_EVP_CIPHER_UPDATE;
        }
        fwrite(out_buf, sizeof(unsigned char), out_len, ofp);
        if (ferror(ofp)) {
            EVP_CIPHER_CTX_free(ctx);
            return errno;
        }
        if (num_bytes_read < BUFSIZE) {
            // reached EOF
            break;
        }
    }

    // cipher final block
    if(!EVP_CipherFinal_ex(ctx, out_buf, &out_len)){
        EVP_CIPHER_CTX_free(ctx);
        return ERR_EVP_CIPHER_FINAL;
    }
    fwrite(out_buf, sizeof(unsigned char), out_len, ofp);
    if (ferror(ofp)) {
        EVP_CIPHER_CTX_free(ctx);
        return errno;
    }
    EVP_CIPHER_CTX_free(ctx);
    return 0;
}

Note, that you dont handle the potentially write error in fwrite.

[matu3ba]

Also fixed some leaks.

[matu3ba]

Remark: EVP_CIPHER_block_size is a simple getter, which should be possible to evaluate at compilation time (ie with constexpr). However, C is suss and instead forces us to use a VLA instead.