* Re: [PATCH 5/6] dmaengine: Add Broadcom SBA RAID driver
From: Anup Patel @ 2017-02-06 12:01 UTC (permalink / raw)
To: Vinod Koul
Cc: Rob Herring, Mark Rutland, Herbert Xu, David S . Miller,
Jassi Brar, Dan Williams, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine, Device Tree,
Linux ARM Kernel, Linux Kernel, linux-crypto, linux-raid
In-Reply-To: <20170205060657.GV19244@localhost>
On Sun, Feb 5, 2017 at 11:36 AM, Vinod Koul <vinod.koul@intel.com> wrote:
> On Thu, Feb 02, 2017 at 10:17:15AM +0530, Anup Patel wrote:
>> +config BCM_SBA_RAID
>> + tristate "Broadcom SBA RAID engine support"
>> + depends on (ARM64 && MAILBOX && RAID6_PQ) || COMPILE_TEST
>> + select DMA_ENGINE
>> + select DMA_ENGINE_RAID
>> + select ASYNC_TX_ENABLE_CHANNEL_SWITCH
>> + default ARCH_BCM_IPROC
>
> whats with the funny alignement?
Sure, I will use tabs here.
>
>> +/* SBA command related defines */
>> +#define SBA_TYPE_SHIFT 48
>> +#define SBA_TYPE_MASK 0x3
>> +#define SBA_TYPE_A 0x0
>> +#define SBA_TYPE_B 0x2
>> +#define SBA_TYPE_C 0x3
>> +#define SBA_USER_DEF_SHIFT 32
>> +#define SBA_USER_DEF_MASK 0xffff
>> +#define SBA_R_MDATA_SHIFT 24
>> +#define SBA_R_MDATA_MASK 0xff
>> +#define SBA_C_MDATA_MS_SHIFT 18
>> +#define SBA_C_MDATA_MS_MASK 0x3
>> +#define SBA_INT_SHIFT 17
>> +#define SBA_INT_MASK 0x1
>> +#define SBA_RESP_SHIFT 16
>> +#define SBA_RESP_MASK 0x1
>> +#define SBA_C_MDATA_SHIFT 8
>> +#define SBA_C_MDATA_MASK 0xff
>> +#define SBA_CMD_SHIFT 0
>> +#define SBA_CMD_MASK 0xf
>> +#define SBA_CMD_ZERO_ALL_BUFFERS 0x8
>> +#define SBA_CMD_LOAD_BUFFER 0x9
>> +#define SBA_CMD_XOR 0xa
>> +#define SBA_CMD_GALOIS_XOR 0xb
>> +#define SBA_CMD_ZERO_BUFFER 0x4
>> +#define SBA_CMD_WRITE_BUFFER 0xc
>
> Try using BIT and GENMAST for hardware descriptions
Sure, will do.
>
>> +
>> +/* SBA C_MDATA helper macros */
>> +#define SBA_C_MDATA_LOAD_VAL(__bnum0) ((__bnum0) & 0x3)
>> +#define SBA_C_MDATA_WRITE_VAL(__bnum0) ((__bnum0) & 0x3)
>> +#define SBA_C_MDATA_XOR_VAL(__bnum1, __bnum0) \
>> + ({ u32 __v = ((__bnum0) & 0x3); \
>> + __v |= ((__bnum1) & 0x3) << 2; \
>> + __v; \
>> + })
>> +#define SBA_C_MDATA_PQ_VAL(__dnum, __bnum1, __bnum0) \
>> + ({ u32 __v = ((__bnum0) & 0x3); \
>> + __v |= ((__bnum1) & 0x3) << 2; \
>> + __v |= ((__dnum) & 0x1f) << 5; \
>> + __v; \
>> + })
>
> ah why are we usig complex macros, why can't these be simple functions..
"static inline functions" seemed too complicated here because most of
these macros are two lines of c-code.
Do you still insist on using "static inline functions"?
>
>> +#define SBA_C_MDATA_LS(__c_mdata_val) ((__c_mdata_val) & 0xff)
>> +#define SBA_C_MDATA_MS(__c_mdata_val) (((__c_mdata_val) >> 8) & 0x3)
>> +
>> +/* Driver helper macros */
>> +#define to_sba_request(tx) \
>> + container_of(tx, struct sba_request, tx)
>> +#define to_sba_device(dchan) \
>> + container_of(dchan, struct sba_device, dma_chan)
>> +
>> +enum sba_request_state {
>> + SBA_REQUEST_STATE_FREE = 1,
>> + SBA_REQUEST_STATE_ALLOCED = 2,
>> + SBA_REQUEST_STATE_PENDING = 3,
>> + SBA_REQUEST_STATE_ACTIVE = 4,
>> + SBA_REQUEST_STATE_COMPLETED = 5,
>> + SBA_REQUEST_STATE_ABORTED = 6,
>
> whats up with a very funny indentation setting, we use 8 chars.
>
> Please re-read the Documentation/process/coding-style.rst
I have double checked this enum. The indentation is fine
and as-per coding style. Am I missing anything else?
>
>> +static int sba_alloc_chan_resources(struct dma_chan *dchan)
>> +{
>> + /*
>> + * We only have one channel so we have pre-alloced
>> + * channel resources. Over here we just return number
>> + * of free request.
>> + */
>> + return sba_free_request_count(to_sba_device(dchan));
>> +}
>
> essentially you are not doing much, so you can skip it. Its an optional
> call.
Sure, will do.
>
>> +static void sba_free_chan_resources(struct dma_chan *dchan)
>> +{
>> + /*
>> + * Channel resources are pre-alloced so we just free-up
>> + * whatever we can so that we can re-use pre-alloced
>> + * channel resources next time.
>> + */
>> + sba_cleanup_inflight_requests(to_sba_device(dchan));
>
> well this one checks for pending requests as well, which shouldn't be there
> when freeing a channel, something seems not quite right here..
>
>> +static int sba_send_mbox_request(struct sba_device *sba,
>> + struct sba_request *req)
>> +{
>> + int mchans_idx, ret = 0;
>> +
>> + /* Select mailbox channel in round-robin fashion */
>> + mchans_idx = atomic_inc_return(&sba->mchans_current);
>> + mchans_idx = mchans_idx % sba->mchans_count;
>> +
>> + /* Send batch message for the request */
>> + req->bmsg.batch.msgs_queued = 0;
>> + ret = mbox_send_message(sba->mchans[mchans_idx], &req->bmsg);
>> + if (ret < 0) {
>> + dev_info(sba->dev, "channel %d message %d (total %d)",
>> + mchans_idx, req->bmsg.batch.msgs_queued,
>> + req->bmsg.batch.msgs_count);
>
> dev_err?
Sure, will use dev_err.
>
>> + dev_err(sba->dev, "send message failed with error %d", ret);
>> + return ret;
>> + }
>> + ret = req->bmsg.error;
>> + if (ret < 0) {
>> + dev_info(sba->dev,
>> + "mbox channel %d message %d (total %d)",
>> + mchans_idx, req->bmsg.batch.msgs_queued,
>> + req->bmsg.batch.msgs_count);
>
> same here
OK.
>
>> +static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
>> +{
>> + unsigned long flags;
>> + dma_cookie_t cookie;
>> + struct sba_request *req;
>> + struct sba_device *sba;
>> +
>> + if (unlikely(!tx))
>> + return -EINVAL;
>> +
>> + sba = to_sba_device(tx->chan);
>> + req = to_sba_request(tx);
>> +
>> + /* Assign cookie and mark request pending */
>> + spin_lock_irqsave(&sba->reqs_lock, flags);
>> + cookie = dma_cookie_assign(tx);
>> + _sba_pending_request(sba, req);
>> + spin_unlock_irqrestore(&sba->reqs_lock, flags);
>> +
>> + /* Try to submit pending request */
>> + sba_issue_pending(&sba->dma_chan);
>
> Nope, thats wrong, caller needs to call .issue_pending for that
This was giving minor performance improvement but I will
remove this since its against API usage.
>
>> +static enum dma_status sba_tx_status(struct dma_chan *dchan,
>> + dma_cookie_t cookie,
>> + struct dma_tx_state *txstate)
>> +{
>> + int mchan_idx;
>> + enum dma_status ret;
>> + struct sba_device *sba = to_sba_device(dchan);
>> +
>> + ret = dma_cookie_status(dchan, cookie, txstate);
>> + if (ret == DMA_COMPLETE)
>> + return ret;
>> +
>> + for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
>> + mbox_client_peek_data(sba->mchans[mchan_idx]);
>
> what is this achieving?
The mbox_client_peek_data() is a hint to mailbox controller driver
to check for available messages.
This gives good performance improvement when some DMA client
code is polling using tx_status() callback.
>
>> +static struct dma_async_tx_descriptor *
>> +sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
>> + size_t len, unsigned long flags)
>> +{
>> + size_t msg_len;
>> + dma_addr_t msg_offset = 0;
>> + unsigned int msgs_count = 0, cmds_count, cmds_idx = 0;
>> + struct sba_device *sba = to_sba_device(dchan);
>> + struct sba_request *req = NULL;
>> +
>> + /* Sanity checks */
>> + if (unlikely(len > sba->req_size))
>> + return NULL;
>
> why is that an error, you can create multiple txn of max length
Sure, I will extend driver to create multiple txn when
"len > req->size"
>
>> +static int sba_async_register(struct sba_device *sba)
>> +{
>> + int ret;
>> + struct dma_device *dma_dev = &sba->dma_dev;
>> +
>> + /* Initialize DMA channel cookie */
>> + sba->dma_chan.device = dma_dev;
>> + dma_cookie_init(&sba->dma_chan);
>> +
>> + /* Initialize DMA device capability mask */
>> + dma_cap_zero(dma_dev->cap_mask);
>> + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
>> + dma_cap_set(DMA_XOR, dma_dev->cap_mask);
>> + dma_cap_set(DMA_PQ, dma_dev->cap_mask);
>> +
>> + /*
>> + * Set mailbox channel device as the base device of
>> + * our dma_device because the actual memory accesses
>> + * will be done by mailbox controller
>> + */
>> + dma_dev->dev = sba->mbox_dev;
>> +
>> + /* Set base prep routines */
>> + dma_dev->device_alloc_chan_resources = sba_alloc_chan_resources;
>> + dma_dev->device_free_chan_resources = sba_free_chan_resources;
>> + dma_dev->device_issue_pending = sba_issue_pending;
>> + dma_dev->device_tx_status = sba_tx_status;
>
> Please add terminate callback support, also add the capabilities, we need to
> advertise that and use in clients
OK, I will add terminate callback.
>
> Also you can simplify bunch of code by using virt-chan support for managing
> channels and descriptors
OK, I will surely explore virt-chan.
Regards,
Anup
^ permalink raw reply
* Re: [PATCH v3 2/3] crypto: ccm - switch to separate cbcmac driver
From: Ard Biesheuvel @ 2017-02-06 10:52 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org, Herbert Xu, Eric Biggers
Cc: linux-arm-kernel@lists.infradead.org, Ard Biesheuvel
In-Reply-To: <1486133377-23290-3-git-send-email-ard.biesheuvel@linaro.org>
On 3 February 2017 at 14:49, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> Update the generic CCM driver to defer CBC-MAC processing to a
> dedicated CBC-MAC ahash transform rather than open coding this
> transform (and much of the associated scatterwalk plumbing) in
> the CCM driver itself.
>
> This cleans up the code considerably, but more importantly, it allows
> the use of alternative CBC-MAC implementations that don't suffer from
> performance degradation due to significant setup time (e.g., the NEON
> based AES code needs to enable/disable the NEON, and load the S-box
> into 16 SIMD registers, which cannot be amortized over the entire input
> when using the cipher interface)
>
> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> ---
> crypto/Kconfig | 1 +
> crypto/ccm.c | 381 +++++++++++++-------
> 2 files changed, 245 insertions(+), 137 deletions(-)
>
> diff --git a/crypto/Kconfig b/crypto/Kconfig
> index 160f08e721cc..e8269d1b0282 100644
> --- a/crypto/Kconfig
> +++ b/crypto/Kconfig
> @@ -263,6 +263,7 @@ comment "Authenticated Encryption with Associated Data"
> config CRYPTO_CCM
> tristate "CCM support"
> select CRYPTO_CTR
> + select CRYPTO_HASH
> select CRYPTO_AEAD
> help
> Support for Counter with CBC MAC. Required for IPsec.
> diff --git a/crypto/ccm.c b/crypto/ccm.c
> index 26b924d1e582..52e307807ff6 100644
> --- a/crypto/ccm.c
> +++ b/crypto/ccm.c
> @@ -11,6 +11,7 @@
> */
>
> #include <crypto/internal/aead.h>
> +#include <crypto/internal/hash.h>
> #include <crypto/internal/skcipher.h>
> #include <crypto/scatterwalk.h>
> #include <linux/err.h>
> @@ -23,11 +24,11 @@
>
> struct ccm_instance_ctx {
> struct crypto_skcipher_spawn ctr;
> - struct crypto_spawn cipher;
> + struct crypto_ahash_spawn mac;
> };
>
> struct crypto_ccm_ctx {
> - struct crypto_cipher *cipher;
> + struct crypto_ahash *mac;
> struct crypto_skcipher *ctr;
> };
>
> @@ -44,15 +45,22 @@ struct crypto_rfc4309_req_ctx {
>
> struct crypto_ccm_req_priv_ctx {
> u8 odata[16];
> - u8 idata[16];
> u8 auth_tag[16];
> - u32 ilen;
> u32 flags;
> struct scatterlist src[3];
> struct scatterlist dst[3];
> struct skcipher_request skreq;
> };
>
> +struct cbcmac_tfm_ctx {
> + struct crypto_cipher *child;
> +};
> +
> +struct cbcmac_desc_ctx {
> + unsigned int len;
> + u8 dg[];
This should be
u8 dg[] CRYPTO_MINALIGN_ATTR;
> +};
> +
> static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx(
> struct aead_request *req)
> {
> @@ -84,7 +92,7 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
> {
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
> struct crypto_skcipher *ctr = ctx->ctr;
> - struct crypto_cipher *tfm = ctx->cipher;
> + struct crypto_ahash *mac = ctx->mac;
> int err = 0;
>
> crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
> @@ -96,11 +104,11 @@ static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key,
> if (err)
> goto out;
>
> - crypto_cipher_clear_flags(tfm, CRYPTO_TFM_REQ_MASK);
> - crypto_cipher_set_flags(tfm, crypto_aead_get_flags(aead) &
> + crypto_ahash_clear_flags(mac, CRYPTO_TFM_REQ_MASK);
> + crypto_ahash_set_flags(mac, crypto_aead_get_flags(aead) &
> CRYPTO_TFM_REQ_MASK);
> - err = crypto_cipher_setkey(tfm, key, keylen);
> - crypto_aead_set_flags(aead, crypto_cipher_get_flags(tfm) &
> + err = crypto_ahash_setkey(mac, key, keylen);
> + crypto_aead_set_flags(aead, crypto_ahash_get_flags(mac) &
> CRYPTO_TFM_RES_MASK);
>
> out:
> @@ -167,119 +175,61 @@ static int format_adata(u8 *adata, unsigned int a)
> return len;
> }
>
> -static void compute_mac(struct crypto_cipher *tfm, u8 *data, int n,
> - struct crypto_ccm_req_priv_ctx *pctx)
> -{
> - unsigned int bs = 16;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> - int datalen, getlen;
> -
> - datalen = n;
> -
> - /* first time in here, block may be partially filled. */
> - getlen = bs - pctx->ilen;
> - if (datalen >= getlen) {
> - memcpy(idata + pctx->ilen, data, getlen);
> - crypto_xor(odata, idata, bs);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> - datalen -= getlen;
> - data += getlen;
> - pctx->ilen = 0;
> - }
> -
> - /* now encrypt rest of data */
> - while (datalen >= bs) {
> - crypto_xor(odata, data, bs);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> -
> - datalen -= bs;
> - data += bs;
> - }
> -
> - /* check and see if there's leftover data that wasn't
> - * enough to fill a block.
> - */
> - if (datalen) {
> - memcpy(idata + pctx->ilen, data, datalen);
> - pctx->ilen += datalen;
> - }
> -}
> -
> -static void get_data_to_compute(struct crypto_cipher *tfm,
> - struct crypto_ccm_req_priv_ctx *pctx,
> - struct scatterlist *sg, unsigned int len)
> -{
> - struct scatter_walk walk;
> - u8 *data_src;
> - int n;
> -
> - scatterwalk_start(&walk, sg);
> -
> - while (len) {
> - n = scatterwalk_clamp(&walk, len);
> - if (!n) {
> - scatterwalk_start(&walk, sg_next(walk.sg));
> - n = scatterwalk_clamp(&walk, len);
> - }
> - data_src = scatterwalk_map(&walk);
> -
> - compute_mac(tfm, data_src, n, pctx);
> - len -= n;
> -
> - scatterwalk_unmap(data_src);
> - scatterwalk_advance(&walk, n);
> - scatterwalk_done(&walk, 0, len);
> - if (len)
> - crypto_yield(pctx->flags);
> - }
> -
> - /* any leftover needs padding and then encrypted */
> - if (pctx->ilen) {
> - int padlen;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> -
> - padlen = 16 - pctx->ilen;
> - memset(idata + pctx->ilen, 0, padlen);
> - crypto_xor(odata, idata, 16);
> - crypto_cipher_encrypt_one(tfm, odata, odata);
> - pctx->ilen = 0;
> - }
> -}
> -
> static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain,
> unsigned int cryptlen)
> {
> + struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
> struct crypto_aead *aead = crypto_aead_reqtfm(req);
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead);
> - struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req);
> - struct crypto_cipher *cipher = ctx->cipher;
> + AHASH_REQUEST_ON_STACK(ahreq, ctx->mac);
> unsigned int assoclen = req->assoclen;
> - u8 *odata = pctx->odata;
> - u8 *idata = pctx->idata;
> - int err;
> + struct scatterlist sg[3];
> + u8 odata[16];
> + u8 idata[16];
> + int ilen, err;
>
> /* format control data for input */
> err = format_input(odata, req, cryptlen);
> if (err)
> goto out;
>
> - /* encrypt first block to use as start in computing mac */
> - crypto_cipher_encrypt_one(cipher, odata, odata);
> + sg_init_table(sg, 3);
> + sg_set_buf(&sg[0], odata, 16);
>
> /* format associated data and compute into mac */
> if (assoclen) {
> - pctx->ilen = format_adata(idata, assoclen);
> - get_data_to_compute(cipher, pctx, req->src, req->assoclen);
> + ilen = format_adata(idata, assoclen);
> + sg_set_buf(&sg[1], idata, ilen);
> + sg_chain(sg, 3, req->src);
> } else {
> - pctx->ilen = 0;
> + ilen = 0;
> + sg_chain(sg, 2, req->src);
> }
>
> - /* compute plaintext into mac */
> - if (cryptlen)
> - get_data_to_compute(cipher, pctx, plain, cryptlen);
> + ahash_request_set_tfm(ahreq, ctx->mac);
> + ahash_request_set_callback(ahreq, pctx->flags, NULL, NULL);
> + ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16);
> + err = crypto_ahash_init(ahreq);
> + if (err)
> + goto out;
> + err = crypto_ahash_update(ahreq);
> + if (err)
> + goto out;
>
> + /* we need to pad the MAC input to a round multiple of the block size */
> + ilen = 16 - (assoclen + ilen) % 16;
> + if (ilen < 16) {
> + memset(idata, 0, ilen);
> + sg_init_table(sg, 2);
> + sg_set_buf(&sg[0], idata, ilen);
> + if (plain)
> + sg_chain(sg, 2, plain);
> + plain = sg;
> + cryptlen += ilen;
> + }
> +
> + ahash_request_set_crypt(ahreq, plain, pctx->odata, cryptlen);
> + err = crypto_ahash_finup(ahreq);
> out:
> return err;
> }
> @@ -453,21 +403,21 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
> struct aead_instance *inst = aead_alg_instance(tfm);
> struct ccm_instance_ctx *ictx = aead_instance_ctx(inst);
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
> - struct crypto_cipher *cipher;
> + struct crypto_ahash *mac;
> struct crypto_skcipher *ctr;
> unsigned long align;
> int err;
>
> - cipher = crypto_spawn_cipher(&ictx->cipher);
> - if (IS_ERR(cipher))
> - return PTR_ERR(cipher);
> + mac = crypto_spawn_ahash(&ictx->mac);
> + if (IS_ERR(mac))
> + return PTR_ERR(mac);
>
> ctr = crypto_spawn_skcipher(&ictx->ctr);
> err = PTR_ERR(ctr);
> if (IS_ERR(ctr))
> - goto err_free_cipher;
> + goto err_free_mac;
>
> - ctx->cipher = cipher;
> + ctx->mac = mac;
> ctx->ctr = ctr;
>
> align = crypto_aead_alignmask(tfm);
> @@ -479,8 +429,8 @@ static int crypto_ccm_init_tfm(struct crypto_aead *tfm)
>
> return 0;
>
> -err_free_cipher:
> - crypto_free_cipher(cipher);
> +err_free_mac:
> + crypto_free_ahash(mac);
> return err;
> }
>
> @@ -488,7 +438,7 @@ static void crypto_ccm_exit_tfm(struct crypto_aead *tfm)
> {
> struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm);
>
> - crypto_free_cipher(ctx->cipher);
> + crypto_free_ahash(ctx->mac);
> crypto_free_skcipher(ctx->ctr);
> }
>
> @@ -496,7 +446,7 @@ static void crypto_ccm_free(struct aead_instance *inst)
> {
> struct ccm_instance_ctx *ctx = aead_instance_ctx(inst);
>
> - crypto_drop_spawn(&ctx->cipher);
> + crypto_drop_ahash(&ctx->mac);
> crypto_drop_skcipher(&ctx->ctr);
> kfree(inst);
> }
> @@ -505,12 +455,13 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> struct rtattr **tb,
> const char *full_name,
> const char *ctr_name,
> - const char *cipher_name)
> + const char *mac_name)
> {
> struct crypto_attr_type *algt;
> struct aead_instance *inst;
> struct skcipher_alg *ctr;
> - struct crypto_alg *cipher;
> + struct crypto_alg *mac_alg;
> + struct hash_alg_common *mac;
> struct ccm_instance_ctx *ictx;
> int err;
>
> @@ -521,25 +472,26 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
> return -EINVAL;
>
> - cipher = crypto_alg_mod_lookup(cipher_name, CRYPTO_ALG_TYPE_CIPHER,
> - CRYPTO_ALG_TYPE_MASK);
> - if (IS_ERR(cipher))
> - return PTR_ERR(cipher);
> + mac_alg = crypto_find_alg(mac_name, &crypto_ahash_type,
> + CRYPTO_ALG_TYPE_HASH,
> + CRYPTO_ALG_TYPE_AHASH_MASK |
> + CRYPTO_ALG_ASYNC);
> + if (IS_ERR(mac_alg))
> + return PTR_ERR(mac_alg);
>
> + mac = __crypto_hash_alg_common(mac_alg);
> err = -EINVAL;
> - if (cipher->cra_blocksize != 16)
> - goto out_put_cipher;
> + if (mac->digestsize != 16)
> + goto out_put_mac;
>
> inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL);
> err = -ENOMEM;
> if (!inst)
> - goto out_put_cipher;
> + goto out_put_mac;
>
> ictx = aead_instance_ctx(inst);
> -
> - err = crypto_init_spawn(&ictx->cipher, cipher,
> - aead_crypto_instance(inst),
> - CRYPTO_ALG_TYPE_MASK);
> + err = crypto_init_ahash_spawn(&ictx->mac, mac,
> + aead_crypto_instance(inst));
> if (err)
> goto err_free_inst;
>
> @@ -548,7 +500,7 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> crypto_requires_sync(algt->type,
> algt->mask));
> if (err)
> - goto err_drop_cipher;
> + goto err_drop_mac;
>
> ctr = crypto_spawn_skcipher_alg(&ictx->ctr);
>
> @@ -564,16 +516,16 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> err = -ENAMETOOLONG;
> if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
> "ccm_base(%s,%s)", ctr->base.cra_driver_name,
> - cipher->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
> + mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
> goto err_drop_ctr;
>
> memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
>
> inst->alg.base.cra_flags = ctr->base.cra_flags & CRYPTO_ALG_ASYNC;
> - inst->alg.base.cra_priority = (cipher->cra_priority +
> + inst->alg.base.cra_priority = (mac->base.cra_priority +
> ctr->base.cra_priority) / 2;
> inst->alg.base.cra_blocksize = 1;
> - inst->alg.base.cra_alignmask = cipher->cra_alignmask |
> + inst->alg.base.cra_alignmask = mac->base.cra_alignmask |
> ctr->base.cra_alignmask |
> (__alignof__(u32) - 1);
> inst->alg.ivsize = 16;
> @@ -593,23 +545,24 @@ static int crypto_ccm_create_common(struct crypto_template *tmpl,
> if (err)
> goto err_drop_ctr;
>
> -out_put_cipher:
> - crypto_mod_put(cipher);
> +out_put_mac:
> + crypto_mod_put(mac_alg);
> return err;
>
> err_drop_ctr:
> crypto_drop_skcipher(&ictx->ctr);
> -err_drop_cipher:
> - crypto_drop_spawn(&ictx->cipher);
> +err_drop_mac:
> + crypto_drop_ahash(&ictx->mac);
> err_free_inst:
> kfree(inst);
> - goto out_put_cipher;
> + goto out_put_mac;
> }
>
> static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
> {
> const char *cipher_name;
> char ctr_name[CRYPTO_MAX_ALG_NAME];
> + char mac_name[CRYPTO_MAX_ALG_NAME];
> char full_name[CRYPTO_MAX_ALG_NAME];
>
> cipher_name = crypto_attr_alg_name(tb[1]);
> @@ -620,12 +573,16 @@ static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb)
> cipher_name) >= CRYPTO_MAX_ALG_NAME)
> return -ENAMETOOLONG;
>
> + if (snprintf(mac_name, CRYPTO_MAX_ALG_NAME, "cbcmac(%s)",
> + cipher_name) >= CRYPTO_MAX_ALG_NAME)
> + return -ENAMETOOLONG;
> +
> if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "ccm(%s)", cipher_name) >=
> CRYPTO_MAX_ALG_NAME)
> return -ENAMETOOLONG;
>
> return crypto_ccm_create_common(tmpl, tb, full_name, ctr_name,
> - cipher_name);
> + mac_name);
> }
>
> static struct crypto_template crypto_ccm_tmpl = {
> @@ -899,14 +856,161 @@ static struct crypto_template crypto_rfc4309_tmpl = {
> .module = THIS_MODULE,
> };
>
> +static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent,
> + const u8 *inkey, unsigned int keylen)
> +{
> + struct cbcmac_tfm_ctx *ctx = crypto_shash_ctx(parent);
> +
> + return crypto_cipher_setkey(ctx->child, inkey, keylen);
> +}
> +
> +static int crypto_cbcmac_digest_init(struct shash_desc *pdesc)
> +{
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + int bs = crypto_shash_digestsize(pdesc->tfm);
> +
> + ctx->len = 0;
> + memset(ctx->dg, 0, bs);
> +
> + return 0;
> +}
> +
> +static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p,
> + unsigned int len)
> +{
> + struct crypto_shash *parent = pdesc->tfm;
> + struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + struct crypto_cipher *tfm = tctx->child;
> + int bs = crypto_shash_digestsize(parent);
> +
> + while (len > 0) {
> + unsigned int l = min(len, bs - ctx->len);
> +
> + crypto_xor(ctx->dg + ctx->len, p, l);
> + ctx->len +=l;
> + len -= l;
> + p += l;
> +
> + if (ctx->len == bs) {
> + crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg);
> + ctx->len = 0;
> + }
> + }
> +
> + return 0;
> +}
> +
> +static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out)
> +{
> + struct crypto_shash *parent = pdesc->tfm;
> + struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent);
> + struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc);
> + struct crypto_cipher *tfm = tctx->child;
> + int bs = crypto_shash_digestsize(parent);
> +
> + if (ctx->len)
> + crypto_cipher_encrypt_one(tfm, out, ctx->dg);
> + else
> + memcpy(out, ctx->dg, bs);
> +
> + return 0;
> +}
> +
> +static int cbcmac_init_tfm(struct crypto_tfm *tfm)
> +{
> + struct crypto_cipher *cipher;
> + struct crypto_instance *inst = (void *)tfm->__crt_alg;
> + struct crypto_spawn *spawn = crypto_instance_ctx(inst);
> + struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
> +
> + cipher = crypto_spawn_cipher(spawn);
> + if (IS_ERR(cipher))
> + return PTR_ERR(cipher);
> +
> + ctx->child = cipher;
> +
> + return 0;
> +};
> +
> +static void cbcmac_exit_tfm(struct crypto_tfm *tfm)
> +{
> + struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
> + crypto_free_cipher(ctx->child);
> +}
> +
> +static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb)
> +{
> + struct shash_instance *inst;
> + struct crypto_alg *alg;
> + int err;
> +
> + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH);
> + if (err)
> + return err;
> +
> + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER,
> + CRYPTO_ALG_TYPE_MASK);
> + if (IS_ERR(alg))
> + return PTR_ERR(alg);
> +
> + inst = shash_alloc_instance("cbcmac", alg);
> + err = PTR_ERR(inst);
> + if (IS_ERR(inst))
> + goto out_put_alg;
> +
> + err = crypto_init_spawn(shash_instance_ctx(inst), alg,
> + shash_crypto_instance(inst),
> + CRYPTO_ALG_TYPE_MASK);
> + if (err)
> + goto out_free_inst;
> +
> + inst->alg.base.cra_priority = alg->cra_priority;
> + inst->alg.base.cra_blocksize = 1;
> +
> + inst->alg.digestsize = alg->cra_blocksize;
> + inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) +
> + alg->cra_blocksize;
> +
> + inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx);
> + inst->alg.base.cra_init = cbcmac_init_tfm;
> + inst->alg.base.cra_exit = cbcmac_exit_tfm;
> +
> + inst->alg.init = crypto_cbcmac_digest_init;
> + inst->alg.update = crypto_cbcmac_digest_update;
> + inst->alg.final = crypto_cbcmac_digest_final;
> + inst->alg.setkey = crypto_cbcmac_digest_setkey;
> +
> + err = shash_register_instance(tmpl, inst);
> +
> +out_free_inst:
> + if (err)
> + shash_free_instance(shash_crypto_instance(inst));
> +
> +out_put_alg:
> + crypto_mod_put(alg);
> + return err;
> +}
> +
> +static struct crypto_template crypto_cbcmac_tmpl = {
> + .name = "cbcmac",
> + .create = cbcmac_create,
> + .free = shash_free_instance,
> + .module = THIS_MODULE,
> +};
> +
> static int __init crypto_ccm_module_init(void)
> {
> int err;
>
> - err = crypto_register_template(&crypto_ccm_base_tmpl);
> + err = crypto_register_template(&crypto_cbcmac_tmpl);
> if (err)
> goto out;
>
> + err = crypto_register_template(&crypto_ccm_base_tmpl);
> + if (err)
> + goto out_undo_cbcmac;
> +
> err = crypto_register_template(&crypto_ccm_tmpl);
> if (err)
> goto out_undo_base;
> @@ -922,6 +1026,8 @@ static int __init crypto_ccm_module_init(void)
> crypto_unregister_template(&crypto_ccm_tmpl);
> out_undo_base:
> crypto_unregister_template(&crypto_ccm_base_tmpl);
> +out_undo_cbcmac:
> + crypto_register_template(&crypto_cbcmac_tmpl);
> goto out;
> }
>
> @@ -930,6 +1036,7 @@ static void __exit crypto_ccm_module_exit(void)
> crypto_unregister_template(&crypto_rfc4309_tmpl);
> crypto_unregister_template(&crypto_ccm_tmpl);
> crypto_unregister_template(&crypto_ccm_base_tmpl);
> + crypto_unregister_template(&crypto_cbcmac_tmpl);
> }
>
> module_init(crypto_ccm_module_init);
> --
> 2.7.4
>
^ permalink raw reply
* Crypto Fixes for 4.10
From: Herbert Xu @ 2017-02-06 9:25 UTC (permalink / raw)
To: Linus Torvalds, David S. Miller, Linux Kernel Mailing List,
Linux Crypto Mailing List
In-Reply-To: <20170201090415.GA32141@gondor.apana.org.au>
Hi Linus:
This push fixes the following issues:
- Use after free in algif_aead.
- Modular aesni regression when pcbc is modular but absent.
- Bug causing IO page faults in ccp.
- Double list add in ccp.
- Null pointer dereference in qat (two patches).
- Panic in chcr.
- Null pointer dereference in chcr.
- Out-of-bound access in chcr.
Please pull from
git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git linus
Gary R Hook (2):
crypto: ccp - Fix DMA operations when IOMMU is enabled
crypto: ccp - Fix double add when creating new DMA command
Giovanni Cabiddu (2):
crypto: qat - fix bar discovery for c62x
crypto: qat - zero esram only for DH85x devices
Harsh Jain (4):
crypto: chcr - Fix panic on dma_unmap_sg
crypto: chcr - Check device is allocated before use
crypto: algif_aead - Fix kernel panic on list_del
crypto: chcr - Fix key length for RFC4106
Herbert Xu (1):
crypto: aesni - Fix failure when pcbc module is absent
arch/x86/crypto/aesni-intel_glue.c | 8 ++--
crypto/algif_aead.c | 2 +-
drivers/crypto/ccp/ccp-dev-v5.c | 2 +-
drivers/crypto/ccp/ccp-dev.h | 1 +
drivers/crypto/ccp/ccp-dmaengine.c | 6 ++-
drivers/crypto/chelsio/chcr_algo.c | 53 +++++++++++----------
drivers/crypto/chelsio/chcr_core.c | 18 ++++---
drivers/crypto/chelsio/chcr_crypto.h | 3 ++
drivers/crypto/qat/qat_c62x/adf_drv.c | 2 +-
drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 +
drivers/crypto/qat/qat_common/qat_hal.c | 4 +-
11 files changed, 55 insertions(+), 45 deletions(-)
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply
* Re: linux-next: build warnings after merge of the crypto tree
From: Herbert Xu @ 2017-02-06 9:03 UTC (permalink / raw)
To: Stephen Rothwell
Cc: linux-next, linux-kernel, Cyrille Pitchen,
Linux Crypto Mailing List
In-Reply-To: <20170206122837.0226fb89@canb.auug.org.au>
Hi Stephen:
On Mon, Feb 06, 2017 at 12:28:37PM +1100, Stephen Rothwell wrote:
>
> After merging the crypto tree, today's linux-next build (x86_64
> allmodconfig) produced these warnings:
>
> warning: (CRYPTO_DEV_ATMEL_AUTHENC) selects CRYPTO_DEV_ATMEL_SHA which has unmet direct dependencies (CRYPTO && CRYPTO_HW && ARCH_AT91)
> warning: (CRYPTO_DEV_ATMEL_AUTHENC) selects CRYPTO_DEV_ATMEL_SHA which has unmet direct dependencies (CRYPTO && CRYPTO_HW && ARCH_AT91)
>
> Introduced by commit
>
> 89a82ef87e01 ("crypto: atmel-authenc - add support to authenc(hmac(shaX), Y(aes)) modes")
>
> In file included from include/linux/printk.h:329:0,
> from include/linux/kernel.h:13,
> from drivers/crypto/atmel-sha.c:17:
This patch should fix both issues. Thanks,
---8<---
Subject: crypto: atmel - Fix authenc compile test warnings
The authenc code depends on the sha code so in order to allow
compile testing on the former we must allow it on sha as well.
This patch enables compile testing on sha as well as tdes and
fixes the few x86-64 warnings that arise in the sha code.
Reported-by: Stephen Rothwell <sfr@canb.auug.org.au>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 7482461..9b7e19e 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -444,7 +444,7 @@ config CRYPTO_DEV_ATMEL_AES
config CRYPTO_DEV_ATMEL_TDES
tristate "Support for Atmel DES/TDES hw accelerator"
- depends on ARCH_AT91
+ depends on ARCH_AT91 || COMPILE_TEST
select CRYPTO_DES
select CRYPTO_BLKCIPHER
help
@@ -457,7 +457,7 @@ config CRYPTO_DEV_ATMEL_TDES
config CRYPTO_DEV_ATMEL_SHA
tristate "Support for Atmel SHA hw accelerator"
- depends on ARCH_AT91
+ depends on ARCH_AT91 || COMPILE_TEST
select CRYPTO_HASH
help
Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512
diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c
index 22d0c0c..b081e78 100644
--- a/drivers/crypto/atmel-sha.c
+++ b/drivers/crypto/atmel-sha.c
@@ -568,7 +568,7 @@ static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
int count, len32;
const u32 *buffer = (const u32 *)buf;
- dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
+ dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %zu, final: %d\n",
ctx->digcnt[1], ctx->digcnt[0], length, final);
atmel_sha_write_ctrl(dd, 0);
@@ -597,7 +597,7 @@ static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
int len32;
- dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
+ dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %zu, final: %d\n",
ctx->digcnt[1], ctx->digcnt[0], length1, final);
len32 = DIV_ROUND_UP(length1, sizeof(u32));
@@ -644,7 +644,7 @@ static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
struct dma_async_tx_descriptor *in_desc;
struct scatterlist sg[2];
- dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
+ dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %zu, final: %d\n",
ctx->digcnt[1], ctx->digcnt[0], length1, final);
dd->dma_lch_in.dma_conf.src_maxburst = 16;
@@ -723,7 +723,7 @@ static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,
ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
- dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen +
+ dev_err(dd->dev, "dma %zu bytes error\n", ctx->buflen +
ctx->block_size);
atmel_sha_complete(dd, -EINVAL);
}
@@ -744,7 +744,7 @@ static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)
final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
- dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n",
+ dev_dbg(dd->dev, "slow: bufcnt: %zu, digcnt: 0x%llx 0x%llx, final: %d\n",
ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final);
if (final)
@@ -772,7 +772,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
if (ctx->bufcnt || ctx->offset)
return atmel_sha_update_dma_slow(dd);
- dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n",
+ dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %zu, total: %u\n",
ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total);
sg = ctx->sg;
@@ -814,7 +814,7 @@ static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
- dev_err(dd->dev, "dma %u bytes error\n",
+ dev_err(dd->dev, "dma %zu bytes error\n",
ctx->buflen + ctx->block_size);
atmel_sha_complete(dd, -EINVAL);
}
@@ -994,7 +994,7 @@ static int atmel_sha_finish(struct ahash_request *req)
if (ctx->digcnt[0] || ctx->digcnt[1])
atmel_sha_copy_ready_hash(req);
- dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1],
+ dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %zu\n", ctx->digcnt[1],
ctx->digcnt[0], ctx->bufcnt);
return 0;
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
^ permalink raw reply related
* Re: [PATCH 1/6] mailbox: Add new API mbox_channel_device() for clients
From: Anup Patel @ 2017-02-06 4:27 UTC (permalink / raw)
To: Jassi Brar
Cc: Vinod Koul, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Dan Williams, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine, Devicetree List,
linux-arm-kernel@lists.infradead.org, Linux Kernel Mailing List,
linux-crypto, linux-raid
In-Reply-To: <CABb+yY1yKnFN-6=+xhZzcYrXNBe5eATthwOoHdvDrLmQzpEZ-Q@mail.gmail.com>
On Fri, Feb 3, 2017 at 5:35 PM, Jassi Brar <jassisinghbrar@gmail.com> wrote:
> On Thu, Feb 2, 2017 at 10:17 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>> The remote processor can have DMAENGINE capabilities and client
>> can pass data to be processed via main memory. In such cases,
>> the client will require DMAble memory for remote processor.
>>
>> This patch adds new API mbox_channel_device() which can be
>> used by clients to get struct device pointer of underlying
>> mailbox controller. This struct device pointer of mailbox
>> controller can be used by clients to allocate DMAble memory
>> for remote processor.
>>
> IIUC, DT already provides a way for what you need.
Thanks for the suggestion. I will explore in this direction and
try to avoid this patch in next revision.
Can you please have a look at FlexRM driver which I
had submitted previously?
https://lkml.org/lkml/2017/1/5/291
https://lkml.org/lkml/2017/1/5/293
Regards,
Anup
^ permalink raw reply
* Re: [PATCH 3/6] async_tx: Handle DMA devices having support for fewer PQ coefficients
From: Anup Patel @ 2017-02-06 3:55 UTC (permalink / raw)
To: Dan Williams
Cc: Vinod Koul, Rob Herring, Mark Rutland, Herbert Xu,
David S . Miller, Jassi Brar, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, BCM Kernel Feedback, dmaengine@vger.kernel.org,
Device Tree, linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, linux-crypto, linux-raid
In-Reply-To: <CAPcyv4icg74StGOjsOpYCDb9x02rrvSrkv6O66WtbSvhOwxYnw@mail.gmail.com>
On Sat, Feb 4, 2017 at 12:12 AM, Dan Williams <dan.j.williams@intel.com> wrote:
> On Fri, Feb 3, 2017 at 2:59 AM, Anup Patel <anup.patel@broadcom.com> wrote:
>>
>>
>> On Thu, Feb 2, 2017 at 11:31 AM, Dan Williams <dan.j.williams@intel.com>
>> wrote:
>>>
>>> On Wed, Feb 1, 2017 at 8:47 PM, Anup Patel <anup.patel@broadcom.com>
>>> wrote:
>>> > The DMAENGINE framework assumes that if PQ offload is supported by a
>>> > DMA device then all 256 PQ coefficients are supported. This assumption
>>> > does not hold anymore because we now have BCM-SBA-RAID offload engine
>>> > which supports PQ offload with limited number of PQ coefficients.
>>> >
>>> > This patch extends async_tx APIs to handle DMA devices with support
>>> > for fewer PQ coefficients.
>>> >
>>> > Signed-off-by: Anup Patel <anup.patel@broadcom.com>
>>> > Reviewed-by: Scott Branden <scott.branden@broadcom.com>
>>> > ---
>>> > crypto/async_tx/async_pq.c | 3 +++
>>> > crypto/async_tx/async_raid6_recov.c | 12 ++++++++++--
>>> > include/linux/dmaengine.h | 19 +++++++++++++++++++
>>> > include/linux/raid/pq.h | 3 +++
>>> > 4 files changed, 35 insertions(+), 2 deletions(-)
>>>
>>> So, I hate the way async_tx does these checks on each operation, and
>>> it's ok for me to say that because it's my fault. Really it's md that
>>> should be validating engine offload capabilities once at the beginning
>>> of time. I'd rather we move in that direction than continue to pile
>>> onto a bad design.
>>
>>
>> Yes, indeed. All async_tx APIs have lot of checks and for high throughput
>> RAID offload engine these checks can add some overhead.
>>
>> I think doing checks in Linux md would be certainly better but this would
>> mean lot of changes in Linux md as well as remove checks in async_tx.
>>
>> Also, async_tx APIs should not find DMA channel on its own instead it
>> should rely on Linux md to provide DMA channel pointer as parameter.
>>
>> It's better to do checks cleanup in async_tx as separate patchset and
>> keep this patchset simple.
>
> That's been the problem with async_tx being broken like this for
> years. Once you get this "small / simple" patch upstream, that
> arguably makes async_tx a little bit worse, there is no longer any
> motivation to fix the underlying issues. If you care about the long
> term health of raid offload and are enabling new hardware support you
> should first tackle the known problems with it before adding new
> features.
Apart from the checks related issue you pointed there are other
issues with async_tx APIs such as:
1. The mechanism to do update PQ (or RAID6 update) operation
in current async_tx APIs is to call async_gen_syndrome() twice
with ASYNC_TX_PQ_XOR_DST flag set. Also, async_gen_syndrome()
will always prefer SW approach when ASYNC_TX_PQ_XOR_DST flag
is set. This means async_tx API is forcing SW approach for update
PQ operation and in-addition we require two async_gen_syndrome()
calls to achieve update PQ. This limitations of async_gen_syndrome()
reduces performance of async_tx APIs. Instead of this we should
have a dedicated async_update_pq() API which will allow RAID
offload engine drivers (such as BCM-FS4-RAID) to implement
update PQ using HW offload and this new API will fall-back to
SW approach using async_gen_syndrome() if no DMA channel
provides update PQ HW offload.
2. In our stress testing, we have observed that dma_map_page()
and dma_unmap_page() used in various async_tx APIs are the
major cause of overhead. If we directly call DMA channel callbacks
with pre-DMA-mapped pages then we get very high throughput.
The async_tx APIs should provide a way for pre-DMA-mapped
pages so that Linux MD can exploit this fact for better performance.
3. We really don't have a test module to stress/benchmark all
async_tx APIs using multi-threading and batching large number
of request in each thread. This kind of test module is very much
required for performance benchmarking and stressing high
throughput (hundreds of Gbps) RAID offload engines (such as
BCM-FS4-RAID).
>From the above, we already have async_tx_test module to
address point3. We also plan to address point1 above but
this would also require changes in Linux MD to use new
async_update_pq() API.
As you can see, this patchset is not end of story of us if we
want best possible utilization of BCM-FS4-RAID.
Regards,
Anup
^ permalink raw reply
* [PATCH v7 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version
From: Sven Schmidt @ 2017-02-05 19:09 UTC (permalink / raw)
To: akpm
Cc: bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck, Sven Schmidt
In-Reply-To: <1486321748-19085-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
This patch updates fs/pstore and fs/squashfs to use the updated
functions from the new LZ4 module.
Signed-off-by: Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
---
fs/pstore/platform.c | 22 +++++++++++++---------
fs/squashfs/lz4_wrapper.c | 12 ++++++------
2 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index 729677e..efab7b6 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -342,31 +342,35 @@ static int compress_lz4(const void *in, void *out, size_t inlen, size_t outlen)
{
int ret;
- ret = lz4_compress(in, inlen, out, &outlen, workspace);
- if (ret) {
- pr_err("lz4_compress error, ret = %d!\n", ret);
+ ret = LZ4_compress_default(in, out, inlen, outlen, workspace);
+ if (!ret) {
+ pr_err("LZ4_compress_default error; compression failed!\n");
return -EIO;
}
- return outlen;
+ return ret;
}
static int decompress_lz4(void *in, void *out, size_t inlen, size_t outlen)
{
int ret;
- ret = lz4_decompress_unknownoutputsize(in, inlen, out, &outlen);
- if (ret) {
- pr_err("lz4_decompress error, ret = %d!\n", ret);
+ ret = LZ4_decompress_safe(in, out, inlen, outlen);
+ if (ret < 0) {
+ /*
+ * LZ4_decompress_safe will return an error code
+ * (< 0) if decompression failed
+ */
+ pr_err("LZ4_decompress_safe error, ret = %d!\n", ret);
return -EIO;
}
- return outlen;
+ return ret;
}
static void allocate_lz4(void)
{
- big_oops_buf_sz = lz4_compressbound(psinfo->bufsize);
+ big_oops_buf_sz = LZ4_compressBound(psinfo->bufsize);
big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL);
if (big_oops_buf) {
workspace = kmalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c
index ff4468b..95da653 100644
--- a/fs/squashfs/lz4_wrapper.c
+++ b/fs/squashfs/lz4_wrapper.c
@@ -97,7 +97,6 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm,
struct squashfs_lz4 *stream = strm;
void *buff = stream->input, *data;
int avail, i, bytes = length, res;
- size_t dest_len = output->length;
for (i = 0; i < b; i++) {
avail = min(bytes, msblk->devblksize - offset);
@@ -108,12 +107,13 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm,
put_bh(bh[i]);
}
- res = lz4_decompress_unknownoutputsize(stream->input, length,
- stream->output, &dest_len);
- if (res)
+ res = LZ4_decompress_safe(stream->input, stream->output,
+ length, output->length);
+
+ if (res < 0)
return -EIO;
- bytes = dest_len;
+ bytes = res;
data = squashfs_first_page(output);
buff = stream->output;
while (data) {
@@ -128,7 +128,7 @@ static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm,
}
squashfs_finish_page(output);
- return dest_len;
+ return res;
}
const struct squashfs_decompressor squashfs_lz4_comp_ops = {
--
2.1.4
^ permalink raw reply related
* [PATCH v7 0/5] Update LZ4 compressor module
From: Sven Schmidt @ 2017-02-05 19:09 UTC (permalink / raw)
To: akpm
Cc: bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck
In-Reply-To: <1482259992-16680-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
This patchset is for updating the LZ4 compression module to a version based
on LZ4 v1.7.3 allowing to use the fast compression algorithm aka LZ4 fast
which provides an "acceleration" parameter as a tradeoff between
high compression ratio and high compression speed.
We want to use LZ4 fast in order to support compression in lustre
and (mostly, based on that) investigate data reduction techniques in behalf of
storage systems.
Also, it will be useful for other users of LZ4 compression, as with LZ4 fast
it is possible to enable applications to use fast and/or high compression
depending on the usecase.
For instance, ZRAM is offering a LZ4 backend and could benefit from an updated
LZ4 in the kernel.
LZ4 homepage: http://www.lz4.org/
LZ4 source repository: https://github.com/lz4/lz4
Source version: 1.7.3
Benchmark (taken from [1], Core i5-4300U @1.9GHz):
----------------|--------------|----------------|----------
Compressor | Compression | Decompression | Ratio
----------------|--------------|----------------|----------
memcpy | 4200 MB/s | 4200 MB/s | 1.000
LZ4 fast 50 | 1080 MB/s | 2650 MB/s | 1.375
LZ4 fast 17 | 680 MB/s | 2220 MB/s | 1.607
LZ4 fast 5 | 475 MB/s | 1920 MB/s | 1.886
LZ4 default | 385 MB/s | 1850 MB/s | 2.101
[1] http://fastcompression.blogspot.de/2015/04/sampling-or-faster-lz4.html
[PATCH 1/5] lib: Update LZ4 compressor module
[PATCH 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version
[PATCH 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
[PATCH 4/5] fs/pstore: fs/squashfs: Change usage of LZ4 to work with new LZ4 version
[PATCH 5/5] lib/lz4: Remove back-compat wrappers
Changes:
v7:
- Fixed errors reported by the Smatch tool
- Changed function documentation comments in lz4.h to match kernel-doc style
- Fixed a misbehaviour of LZ4HC caused by the wrong level of indentation
concerning two for loops introduced after I refactored the code style using
checkpatch.pl (upstream LZ4 put dozens of stuff in just one line, gnah)
- Updated the crypto tests for LZ4 since they did fail for the new code
and hence zram did fail to allocate memory for LZ4
v6:
- Fixed LZ4_NBCOMMONBYTES() for 64-bit little endian
- Reset LZ4_MEMORY_USAGE to 14 (which is the value used in
upstream LZ4 as well as the previous kernel module)
- Fixed that weird double-indentation in lz4defs.h and lz4.h
- Adjusted general styling issues in lz4defs.h
(e.g. lines consisting of more than one instruction)
- Removed the architecture-dependent typedef to reg_t
since upstream LZ4 is just using size_t and that works fine
- Changed error messages in pstore/platform.c:
* LZ4_compress_default always returns 0 in case of an error
(no need to print the return value)
* LZ4_decompress_safe returns a negative error message
(return value _does_ matter)
v5:
- Added a fifth patch to remove the back-compat wrappers introduced
to ensure bisectibility between the patches (the functions are no longer
needed since there's no callers left)
v4:
- Fixed kbuild errors
- Re-added lz4_compressbound as alias for LZ4_compressBound
to ensure backwards compatibility
- Wrapped LZ4_hash5 with check for LZ4_ARCH64 since it is only used there
and triggers an unused function warning when false
v3:
- Adjusted the code to satisfy kernel coding style (checkpatch.pl)
- Made sure the changes to LZ4 in Kernel (overflow checks etc.)
are included in the new module (they are)
- Removed the second LZ4_compressBound function with related name but
different return type
- Corrected version number (was LZ4 1.7.3)
- Added missing LZ4 streaming functions
v2:
- Changed order of the patches since in the initial patchset the lz4.h was in the
last patch but was referenced by the other ones
- Split lib/decompress_unlz4.c in an own patch
- Fixed errors reported by the buildbot
- Further refactorings
- Added more appropriate copyright note to include/linux/lz4.h
^ permalink raw reply
* [PATCH v7 5/5] lib/lz4: Remove back-compat wrappers
From: Sven Schmidt @ 2017-02-05 19:09 UTC (permalink / raw)
To: akpm
Cc: bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck, Sven Schmidt
In-Reply-To: <1486321748-19085-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
This patch removes the functions introduced as wrappers for providing
backwards compatibility to the prior LZ4 version.
They're not needed anymore since there's no callers left.
Signed-off-by: Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
---
include/linux/lz4.h | 69 ------------------------------------------------
lib/lz4/lz4_compress.c | 22 ---------------
lib/lz4/lz4_decompress.c | 42 -----------------------------
lib/lz4/lz4hc_compress.c | 23 ----------------
4 files changed, 156 deletions(-)
diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index 1b7ab2a..a3912d7 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -173,18 +173,6 @@ static inline int LZ4_compressBound(size_t isize)
}
/**
- * lz4_compressbound() - For backwards compatibility; see LZ4_compressBound
- * @isize: Size of the input data
- *
- * Return: Max. size LZ4 may output in a "worst case" szenario
- * (data not compressible)
- */
-static inline int lz4_compressbound(size_t isize)
-{
- return LZ4_COMPRESSBOUND(isize);
-}
-
-/**
* LZ4_compress_default() - Compress data from source to dest
* @source: source address of the original data
* @dest: output buffer address of the compressed data
@@ -257,20 +245,6 @@ int LZ4_compress_fast(const char *source, char *dest, int inputSize,
int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr,
int targetDestSize, void *wrkmem);
-/*
- * lz4_compress() - For backward compatibility, see LZ4_compress_default
- * @src: source address of the original data
- * @src_len: size of the original data
- * @dst: output buffer address of the compressed data. This requires 'dst'
- * of size LZ4_COMPRESSBOUND
- * @dst_len: is the output size, which is returned after compress done
- * @workmem: address of the working memory.
- *
- * Return: Success if return 0, Error if return < 0
- */
-int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
- size_t *dst_len, void *wrkmem);
-
/*-************************************************************************
* Decompression Functions
**************************************************************************/
@@ -346,34 +320,6 @@ int LZ4_decompress_safe(const char *source, char *dest, int compressedSize,
int LZ4_decompress_safe_partial(const char *source, char *dest,
int compressedSize, int targetOutputSize, int maxDecompressedSize);
-/*
- * lz4_decompress_unknownoutputsize() - For backwards compatibility,
- * see LZ4_decompress_safe
- * @src: source address of the compressed data
- * @src_len: is the input size, therefore the compressed size
- * @dest: output buffer address of the decompressed data
- * which must be already allocated
- * @dest_len: is the max size of the destination buffer, which is
- * returned with actual size of decompressed data after decompress done
- *
- * Return: Success if return 0, Error if return (< 0)
- */
-int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
- unsigned char *dest, size_t *dest_len);
-
-/**
- * lz4_decompress() - For backwards cocmpatibility, see LZ4_decompress_fast
- * @src: source address of the compressed data
- * @src_len: is the input size, which is returned after decompress done
- * @dest: output buffer address of the decompressed data,
- * which must be already allocated
- * @actual_dest_len: is the size of uncompressed data, supposing it's known
- *
- * Return: Success if return 0, Error if return (< 0)
- */
-int lz4_decompress(const unsigned char *src, size_t *src_len,
- unsigned char *dest, size_t actual_dest_len);
-
/*-************************************************************************
* LZ4 HC Compression
**************************************************************************/
@@ -401,21 +347,6 @@ int LZ4_compress_HC(const char *src, char *dst, int srcSize, int dstCapacity,
int compressionLevel, void *wrkmem);
/**
- * lz4hc_compress() - For backwards compatibility, see LZ4_compress_HC
- * @src: source address of the original data
- * @src_len: size of the original data
- * @dst: output buffer address of the compressed data. This requires 'dst'
- * of size LZ4_COMPRESSBOUND.
- * @dst_len: is the output size, which is returned after compress done
- * @wrkmem: address of the working memory.
- * This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
- *
- * Return : Success if return 0, Error if return (< 0)
- */
-int lz4hc_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
- size_t *dst_len, void *wrkmem);
-
-/**
* LZ4_resetStreamHC() - Init an allocated 'LZ4_streamHC_t' structure
* @streamHCPtr: pointer to the 'LZ4_streamHC_t' structure
* @compressionLevel: Recommended values are between 4 and 9, although any
diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c
index 6aa7ac3..697dbda 100644
--- a/lib/lz4/lz4_compress.c
+++ b/lib/lz4/lz4_compress.c
@@ -883,27 +883,5 @@ int LZ4_compress_fast_continue(LZ4_stream_t *LZ4_stream, const char *source,
}
EXPORT_SYMBOL(LZ4_compress_fast_continue);
-/*-******************************
- * For backwards compatibility
- ********************************/
-int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
- size_t *dst_len, void *wrkmem) {
- *dst_len = LZ4_compress_default(src, dst, src_len,
- *dst_len, wrkmem);
-
- /*
- * Prior lz4_compress will return -1 in case of error
- * and 0 on success
- * while new LZ4_compress_fast/default
- * returns 0 in case of error
- * and the output length on success
- */
- if (!*dst_len)
- return -1;
- else
- return 0;
-}
-EXPORT_SYMBOL(lz4_compress);
-
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("LZ4 compressor");
diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c
index fd665ca..9bf9182 100644
--- a/lib/lz4/lz4_decompress.c
+++ b/lib/lz4/lz4_decompress.c
@@ -483,47 +483,5 @@ int LZ4_decompress_fast_usingDict(const char *source, char *dest,
}
EXPORT_SYMBOL(LZ4_decompress_fast_usingDict);
-/*-******************************
- * For backwards compatibility
- ********************************/
-int lz4_decompress_unknownoutputsize(const unsigned char *src,
- size_t src_len, unsigned char *dest, size_t *dest_len) {
- *dest_len = LZ4_decompress_safe(src, dest,
- src_len, *dest_len);
-
- /*
- * Prior lz4_decompress_unknownoutputsize will return
- * 0 for success and a negative result for error
- * new LZ4_decompress_safe returns
- * - the length of data read on success
- * - and also a negative result on error
- * meaning when result > 0, we just return 0 here
- */
- if (src_len > 0)
- return 0;
- else
- return -1;
-}
-EXPORT_SYMBOL(lz4_decompress_unknownoutputsize);
-
-int lz4_decompress(const unsigned char *src, size_t *src_len,
- unsigned char *dest, size_t actual_dest_len) {
- *src_len = LZ4_decompress_fast(src, dest, actual_dest_len);
-
- /*
- * Prior lz4_decompress will return
- * 0 for success and a negative result for error
- * new LZ4_decompress_fast returns
- * - the length of data read on success
- * - and also a negative result on error
- * meaning when result > 0, we just return 0 here
- */
- if (*src_len > 0)
- return 0;
- else
- return -1;
-}
-EXPORT_SYMBOL(lz4_decompress);
-
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("LZ4 decompressor");
diff --git a/lib/lz4/lz4hc_compress.c b/lib/lz4/lz4hc_compress.c
index f1d4662..8363292 100644
--- a/lib/lz4/lz4hc_compress.c
+++ b/lib/lz4/lz4hc_compress.c
@@ -765,28 +765,5 @@ int LZ4_saveDictHC(
}
EXPORT_SYMBOL(LZ4_saveDictHC);
-/*-******************************
- * For backwards compatibility
- ********************************/
-int lz4hc_compress(const unsigned char *src, size_t src_len,
- unsigned char *dst, size_t *dst_len, void *wrkmem)
-{
- *dst_len = LZ4_compress_HC(src, dst, src_len,
- *dst_len, LZ4HC_DEFAULT_CLEVEL, wrkmem);
-
- /*
- * Prior lz4hc_compress will return -1 in case of error
- * and 0 on success
- * while new LZ4_compress_HC
- * returns 0 in case of error
- * and the output length on success
- */
- if (!*dst_len)
- return -1;
- else
- return 0;
-}
-EXPORT_SYMBOL(lz4hc_compress);
-
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("LZ4 HC compressor");
--
2.1.4
^ permalink raw reply related
* [PATCH v7 3/5] crypto: Change LZ4 modules to work with new LZ4 module version
From: Sven Schmidt @ 2017-02-05 19:09 UTC (permalink / raw)
To: akpm
Cc: bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck, Sven Schmidt
In-Reply-To: <1486321748-19085-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
This patch updates the crypto modules using LZ4 compression as well as the
test cases in testmgr.h to work with the new LZ4 module version.
Signed-off-by: Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
---
crypto/lz4.c | 23 ++++-----
crypto/lz4hc.c | 23 ++++-----
crypto/testmgr.h | 142 +++++++++++++++++++++++++++++++++++++++----------------
3 files changed, 120 insertions(+), 68 deletions(-)
diff --git a/crypto/lz4.c b/crypto/lz4.c
index 99c1b2c..71eff9b 100644
--- a/crypto/lz4.c
+++ b/crypto/lz4.c
@@ -66,15 +66,13 @@ static void lz4_exit(struct crypto_tfm *tfm)
static int __lz4_compress_crypto(const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen, void *ctx)
{
- size_t tmp_len = *dlen;
- int err;
+ int out_len = LZ4_compress_default(src, dst,
+ slen, *dlen, ctx);
- err = lz4_compress(src, slen, dst, &tmp_len, ctx);
-
- if (err < 0)
+ if (!out_len)
return -EINVAL;
- *dlen = tmp_len;
+ *dlen = out_len;
return 0;
}
@@ -96,16 +94,13 @@ static int lz4_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
static int __lz4_decompress_crypto(const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen, void *ctx)
{
- int err;
- size_t tmp_len = *dlen;
- size_t __slen = slen;
+ int out_len = LZ4_decompress_safe(src, dst, slen, *dlen);
- err = lz4_decompress_unknownoutputsize(src, __slen, dst, &tmp_len);
- if (err < 0)
- return -EINVAL;
+ if (out_len < 0)
+ return out_len;
- *dlen = tmp_len;
- return err;
+ *dlen = out_len;
+ return 0;
}
static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c
index 75ffc4a..03a34a8 100644
--- a/crypto/lz4hc.c
+++ b/crypto/lz4hc.c
@@ -65,15 +65,13 @@ static void lz4hc_exit(struct crypto_tfm *tfm)
static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen, void *ctx)
{
- size_t tmp_len = *dlen;
- int err;
+ int out_len = LZ4_compress_HC(src, dst, slen,
+ *dlen, LZ4HC_DEFAULT_CLEVEL, ctx);
- err = lz4hc_compress(src, slen, dst, &tmp_len, ctx);
-
- if (err < 0)
+ if (!out_len)
return -EINVAL;
- *dlen = tmp_len;
+ *dlen = out_len;
return 0;
}
@@ -97,16 +95,13 @@ static int lz4hc_compress_crypto(struct crypto_tfm *tfm, const u8 *src,
static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen,
u8 *dst, unsigned int *dlen, void *ctx)
{
- int err;
- size_t tmp_len = *dlen;
- size_t __slen = slen;
+ int out_len = LZ4_decompress_safe(src, dst, slen, *dlen);
- err = lz4_decompress_unknownoutputsize(src, __slen, dst, &tmp_len);
- if (err < 0)
- return -EINVAL;
+ if (out_len < 0)
+ return out_len;
- *dlen = tmp_len;
- return err;
+ *dlen = out_len;
+ return 0;
}
static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src,
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 9b656be..98d4be0 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -34498,31 +34498,62 @@ static struct hash_testvec bfin_crc_tv_template[] = {
static struct comp_testvec lz4_comp_tv_template[] = {
{
- .inlen = 70,
- .outlen = 45,
- .input = "Join us now and share the software "
- "Join us now and share the software ",
- .output = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
- "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
- "\x64\x20\x73\x68\x61\x72\x65\x20"
- "\x74\x68\x65\x20\x73\x6f\x66\x74"
- "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
- "\x77\x61\x72\x65\x20",
+ .inlen = 255,
+ .outlen = 218,
+ .input = "LZ4 is lossless compression algorithm, providing"
+ " compression speed at 400 MB/s per core, scalable "
+ "with multi-cores CPU. It features an extremely fast "
+ "decoder, with speed in multiple GB/s per core, "
+ "typically reaching RAM speed limits on multi-core "
+ "systems.",
+ .output = "\xf9\x21\x4c\x5a\x34\x20\x69\x73\x20\x6c\x6f\x73\x73"
+ "\x6c\x65\x73\x73\x20\x63\x6f\x6d\x70\x72\x65\x73\x73"
+ "\x69\x6f\x6e\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d"
+ "\x2c\x20\x70\x72\x6f\x76\x69\x64\x69\x6e\x67\x21\x00"
+ "\xf0\x21\x73\x70\x65\x65\x64\x20\x61\x74\x20\x34\x30"
+ "\x30\x20\x4d\x42\x2f\x73\x20\x70\x65\x72\x20\x63\x6f"
+ "\x72\x65\x2c\x20\x73\x63\x61\x6c\x61\x62\x6c\x65\x20"
+ "\x77\x69\x74\x68\x20\x6d\x75\x6c\x74\x69\x2d\x1a\x00"
+ "\xf0\x00\x73\x20\x43\x50\x55\x2e\x20\x49\x74\x20\x66"
+ "\x65\x61\x74\x75\x11\x00\xf2\x0b\x61\x6e\x20\x65\x78"
+ "\x74\x72\x65\x6d\x65\x6c\x79\x20\x66\x61\x73\x74\x20"
+ "\x64\x65\x63\x6f\x64\x65\x72\x2c\x3d\x00\x02\x67\x00"
+ "\x22\x69\x6e\x46\x00\x5a\x70\x6c\x65\x20\x47\x6c\x00"
+ "\xf0\x00\x74\x79\x70\x69\x63\x61\x6c\x6c\x79\x20\x72"
+ "\x65\x61\x63\x68\xa7\x00\x33\x52\x41\x4d\x38\x00\x83"
+ "\x6c\x69\x6d\x69\x74\x73\x20\x6f\x3f\x00\x01\x85\x00"
+ "\x90\x20\x73\x79\x73\x74\x65\x6d\x73\x2e",
+
},
};
static struct comp_testvec lz4_decomp_tv_template[] = {
{
- .inlen = 45,
- .outlen = 70,
- .input = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
- "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
- "\x64\x20\x73\x68\x61\x72\x65\x20"
- "\x74\x68\x65\x20\x73\x6f\x66\x74"
- "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
- "\x77\x61\x72\x65\x20",
- .output = "Join us now and share the software "
- "Join us now and share the software ",
+ .inlen = 218,
+ .outlen = 255,
+ .input = "\xf9\x21\x4c\x5a\x34\x20\x69\x73\x20\x6c\x6f\x73\x73"
+ "\x6c\x65\x73\x73\x20\x63\x6f\x6d\x70\x72\x65\x73\x73"
+ "\x69\x6f\x6e\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d"
+ "\x2c\x20\x70\x72\x6f\x76\x69\x64\x69\x6e\x67\x21\x00"
+ "\xf0\x21\x73\x70\x65\x65\x64\x20\x61\x74\x20\x34\x30"
+ "\x30\x20\x4d\x42\x2f\x73\x20\x70\x65\x72\x20\x63\x6f"
+ "\x72\x65\x2c\x20\x73\x63\x61\x6c\x61\x62\x6c\x65\x20"
+ "\x77\x69\x74\x68\x20\x6d\x75\x6c\x74\x69\x2d\x1a\x00"
+ "\xf0\x00\x73\x20\x43\x50\x55\x2e\x20\x49\x74\x20\x66"
+ "\x65\x61\x74\x75\x11\x00\xf2\x0b\x61\x6e\x20\x65\x78"
+ "\x74\x72\x65\x6d\x65\x6c\x79\x20\x66\x61\x73\x74\x20"
+ "\x64\x65\x63\x6f\x64\x65\x72\x2c\x3d\x00\x02\x67\x00"
+ "\x22\x69\x6e\x46\x00\x5a\x70\x6c\x65\x20\x47\x6c\x00"
+ "\xf0\x00\x74\x79\x70\x69\x63\x61\x6c\x6c\x79\x20\x72"
+ "\x65\x61\x63\x68\xa7\x00\x33\x52\x41\x4d\x38\x00\x83"
+ "\x6c\x69\x6d\x69\x74\x73\x20\x6f\x3f\x00\x01\x85\x00"
+ "\x90\x20\x73\x79\x73\x74\x65\x6d\x73\x2e",
+ .output = "LZ4 is lossless compression algorithm, providing"
+ " compression speed at 400 MB/s per core, scalable "
+ "with multi-cores CPU. It features an extremely fast "
+ "decoder, with speed in multiple GB/s per core, "
+ "typically reaching RAM speed limits on multi-core "
+ "systems.",
},
};
@@ -34531,31 +34562,62 @@ static struct comp_testvec lz4_decomp_tv_template[] = {
static struct comp_testvec lz4hc_comp_tv_template[] = {
{
- .inlen = 70,
- .outlen = 45,
- .input = "Join us now and share the software "
- "Join us now and share the software ",
- .output = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
- "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
- "\x64\x20\x73\x68\x61\x72\x65\x20"
- "\x74\x68\x65\x20\x73\x6f\x66\x74"
- "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
- "\x77\x61\x72\x65\x20",
+ .inlen = 255,
+ .outlen = 216,
+ .input = "LZ4 is lossless compression algorithm, providing"
+ " compression speed at 400 MB/s per core, scalable "
+ "with multi-cores CPU. It features an extremely fast "
+ "decoder, with speed in multiple GB/s per core, "
+ "typically reaching RAM speed limits on multi-core "
+ "systems.",
+ .output = "\xf9\x21\x4c\x5a\x34\x20\x69\x73\x20\x6c\x6f\x73\x73"
+ "\x6c\x65\x73\x73\x20\x63\x6f\x6d\x70\x72\x65\x73\x73"
+ "\x69\x6f\x6e\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d"
+ "\x2c\x20\x70\x72\x6f\x76\x69\x64\x69\x6e\x67\x21\x00"
+ "\xf0\x21\x73\x70\x65\x65\x64\x20\x61\x74\x20\x34\x30"
+ "\x30\x20\x4d\x42\x2f\x73\x20\x70\x65\x72\x20\x63\x6f"
+ "\x72\x65\x2c\x20\x73\x63\x61\x6c\x61\x62\x6c\x65\x20"
+ "\x77\x69\x74\x68\x20\x6d\x75\x6c\x74\x69\x2d\x1a\x00"
+ "\xf0\x00\x73\x20\x43\x50\x55\x2e\x20\x49\x74\x20\x66"
+ "\x65\x61\x74\x75\x11\x00\xf2\x0b\x61\x6e\x20\x65\x78"
+ "\x74\x72\x65\x6d\x65\x6c\x79\x20\x66\x61\x73\x74\x20"
+ "\x64\x65\x63\x6f\x64\x65\x72\x2c\x3d\x00\x02\x67\x00"
+ "\x22\x69\x6e\x46\x00\x5a\x70\x6c\x65\x20\x47\x6c\x00"
+ "\xf0\x00\x74\x79\x70\x69\x63\x61\x6c\x6c\x79\x20\x72"
+ "\x65\x61\x63\x68\xa7\x00\x33\x52\x41\x4d\x38\x00\x97"
+ "\x6c\x69\x6d\x69\x74\x73\x20\x6f\x6e\x85\x00\x90\x20"
+ "\x73\x79\x73\x74\x65\x6d\x73\x2e",
+
},
};
static struct comp_testvec lz4hc_decomp_tv_template[] = {
{
- .inlen = 45,
- .outlen = 70,
- .input = "\xf0\x10\x4a\x6f\x69\x6e\x20\x75"
- "\x73\x20\x6e\x6f\x77\x20\x61\x6e"
- "\x64\x20\x73\x68\x61\x72\x65\x20"
- "\x74\x68\x65\x20\x73\x6f\x66\x74"
- "\x77\x0d\x00\x0f\x23\x00\x0b\x50"
- "\x77\x61\x72\x65\x20",
- .output = "Join us now and share the software "
- "Join us now and share the software ",
+ .inlen = 216,
+ .outlen = 255,
+ .input = "\xf9\x21\x4c\x5a\x34\x20\x69\x73\x20\x6c\x6f\x73\x73"
+ "\x6c\x65\x73\x73\x20\x63\x6f\x6d\x70\x72\x65\x73\x73"
+ "\x69\x6f\x6e\x20\x61\x6c\x67\x6f\x72\x69\x74\x68\x6d"
+ "\x2c\x20\x70\x72\x6f\x76\x69\x64\x69\x6e\x67\x21\x00"
+ "\xf0\x21\x73\x70\x65\x65\x64\x20\x61\x74\x20\x34\x30"
+ "\x30\x20\x4d\x42\x2f\x73\x20\x70\x65\x72\x20\x63\x6f"
+ "\x72\x65\x2c\x20\x73\x63\x61\x6c\x61\x62\x6c\x65\x20"
+ "\x77\x69\x74\x68\x20\x6d\x75\x6c\x74\x69\x2d\x1a\x00"
+ "\xf0\x00\x73\x20\x43\x50\x55\x2e\x20\x49\x74\x20\x66"
+ "\x65\x61\x74\x75\x11\x00\xf2\x0b\x61\x6e\x20\x65\x78"
+ "\x74\x72\x65\x6d\x65\x6c\x79\x20\x66\x61\x73\x74\x20"
+ "\x64\x65\x63\x6f\x64\x65\x72\x2c\x3d\x00\x02\x67\x00"
+ "\x22\x69\x6e\x46\x00\x5a\x70\x6c\x65\x20\x47\x6c\x00"
+ "\xf0\x00\x74\x79\x70\x69\x63\x61\x6c\x6c\x79\x20\x72"
+ "\x65\x61\x63\x68\xa7\x00\x33\x52\x41\x4d\x38\x00\x97"
+ "\x6c\x69\x6d\x69\x74\x73\x20\x6f\x6e\x85\x00\x90\x20"
+ "\x73\x79\x73\x74\x65\x6d\x73\x2e",
+ .output = "LZ4 is lossless compression algorithm, providing"
+ " compression speed at 400 MB/s per core, scalable "
+ "with multi-cores CPU. It features an extremely fast "
+ "decoder, with speed in multiple GB/s per core, "
+ "typically reaching RAM speed limits on multi-core "
+ "systems.",
},
};
--
2.1.4
^ permalink raw reply related
* [PATCH v7 2/5] lib/decompress_unlz4: Change module to work with new LZ4 module version
From: Sven Schmidt @ 2017-02-05 19:09 UTC (permalink / raw)
To: akpm
Cc: bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck, Sven Schmidt
In-Reply-To: <1486321748-19085-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
This patch updates the unlz4 wrapper to work with the
updated LZ4 kernel module version.
Signed-off-by: Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
---
lib/decompress_unlz4.c | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c
index 036fc88..1b0baf3 100644
--- a/lib/decompress_unlz4.c
+++ b/lib/decompress_unlz4.c
@@ -72,7 +72,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
error("NULL input pointer and missing fill function");
goto exit_1;
} else {
- inp = large_malloc(lz4_compressbound(uncomp_chunksize));
+ inp = large_malloc(LZ4_compressBound(uncomp_chunksize));
if (!inp) {
error("Could not allocate input buffer");
goto exit_1;
@@ -136,7 +136,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
inp += 4;
size -= 4;
} else {
- if (chunksize > lz4_compressbound(uncomp_chunksize)) {
+ if (chunksize > LZ4_compressBound(uncomp_chunksize)) {
error("chunk length is longer than allocated");
goto exit_2;
}
@@ -152,11 +152,14 @@ STATIC inline int INIT unlz4(u8 *input, long in_len,
out_len -= dest_len;
} else
dest_len = out_len;
- ret = lz4_decompress(inp, &chunksize, outp, dest_len);
+
+ ret = LZ4_decompress_fast(inp, outp, dest_len);
+ chunksize = ret;
#else
dest_len = uncomp_chunksize;
- ret = lz4_decompress_unknownoutputsize(inp, chunksize, outp,
- &dest_len);
+
+ ret = LZ4_decompress_safe(inp, outp, chunksize, dest_len);
+ dest_len = ret;
#endif
if (ret < 0) {
error("Decoding failed");
--
2.1.4
^ permalink raw reply related
* [PATCH v7 1/5] lib: Update LZ4 compressor module
From: Sven Schmidt @ 2017-02-05 19:09 UTC (permalink / raw)
To: akpm
Cc: bongkyu.kim, rsalvaterra, sergey.senozhatsky, gregkh,
linux-kernel, herbert, davem, linux-crypto, anton, ccross,
keescook, tony.luck, Sven Schmidt
In-Reply-To: <1486321748-19085-1-git-send-email-4sschmid@informatik.uni-hamburg.de>
This patch updates LZ4 kernel module to LZ4 v1.7.3 by Yann Collet.
The kernel module is inspired by the previous work by Chanho Min.
The updated LZ4 module will not break existing code since the patchset
contains appropriate changes.
API changes:
New method LZ4_compress_fast which differs from the variant available
in kernel by the new acceleration parameter,
allowing to trade compression ratio for more compression speed
and vice versa.
LZ4_decompress_fast is the respective decompression method,
featuring a very fast decoder (multiple GB/s per core),
able to reach RAM speed in multi-core systems.
The decompressor allows to decompress data compressed with
LZ4 fast as well as the LZ4 HC (high compression) algorithm.
Also the useful functions LZ4_decompress_safe_partial and
LZ4_compress_destsize were added.
The latter reverses the logic by trying to compress as much
data as possible from source to dest while the former aims
to decompress partial blocks of data.
A bunch of streaming functions were also added
which allow compressig/decompressing data in multiple steps
(so called "streaming mode").
The methods lz4_compress and lz4_decompress_unknownoutputsize
are now known as LZ4_compress_default respectivley
LZ4_decompress_safe.
The old methods will be removed since there's
no callers left in the code.
Signed-off-by: Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
---
include/linux/lz4.h | 762 ++++++++++++++++++++++++++++---
lib/lz4/lz4_compress.c | 1114 ++++++++++++++++++++++++++++++++--------------
lib/lz4/lz4_decompress.c | 696 ++++++++++++++++++-----------
lib/lz4/lz4defs.h | 334 ++++++++------
lib/lz4/lz4hc_compress.c | 867 +++++++++++++++++++++++-------------
5 files changed, 2690 insertions(+), 1083 deletions(-)
diff --git a/include/linux/lz4.h b/include/linux/lz4.h
index 6b784c5..1b7ab2a 100644
--- a/include/linux/lz4.h
+++ b/include/linux/lz4.h
@@ -1,87 +1,717 @@
-#ifndef __LZ4_H__
-#define __LZ4_H__
-/*
- * LZ4 Kernel Interface
+/* LZ4 Kernel Interface
*
* Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
+ * Copyright (C) 2016, Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
+ *
+ * This file is based on the original header file
+ * for LZ4 - Fast LZ compression algorithm.
+ *
+ * LZ4 - Fast LZ compression algorithm
+ * Copyright (C) 2011-2016, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
*/
-#define LZ4_MEM_COMPRESS (16384)
-#define LZ4HC_MEM_COMPRESS (262144 + (2 * sizeof(unsigned char *)))
+#ifndef __LZ4_H__
+#define __LZ4_H__
+
+#include <linux/types.h>
+#include <linux/string.h> /* memset, memcpy */
+
+/*-************************************************************************
+ * CONSTANTS
+ **************************************************************************/
/*
- * lz4_compressbound()
- * Provides the maximum size that LZ4 may output in a "worst case" scenario
- * (input data not compressible)
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes
+ * (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
-static inline size_t lz4_compressbound(size_t isize)
+#define LZ4_MEMORY_USAGE 14
+
+#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize) (\
+ (unsigned int)(isize) > (unsigned int)LZ4_MAX_INPUT_SIZE \
+ ? 0 \
+ : (isize) + ((isize)/255) + 16)
+
+#define LZ4_ACCELERATION_DEFAULT 1
+#define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2)
+#define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE)
+#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG)
+
+#define LZ4HC_MIN_CLEVEL 3
+#define LZ4HC_DEFAULT_CLEVEL 9
+#define LZ4HC_MAX_CLEVEL 16
+
+#define LZ4HC_DICTIONARY_LOGSIZE 16
+#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
+#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
+#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE - 1)
+#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG)
+#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
+
+/*-************************************************************************
+ * STREAMING CONSTANTS AND STRUCTURES
+ **************************************************************************/
+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long))
+
+#define LZ4_STREAMHCSIZE 262192
+#define LZ4_STREAMHCSIZE_SIZET (262192 / sizeof(size_t))
+
+#define LZ4_STREAMDECODESIZE_U64 4
+#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * \
+ sizeof(unsigned long long))
+
+/*
+ * LZ4_stream_t - information structure to track an LZ4 stream.
+ */
+typedef struct {
+ uint32_t hashTable[LZ4_HASH_SIZE_U32];
+ uint32_t currentOffset;
+ uint32_t initCheck;
+ const uint8_t *dictionary;
+ uint8_t *bufferStart;
+ uint32_t dictSize;
+} LZ4_stream_t_internal;
+typedef union {
+ unsigned long long table[LZ4_STREAMSIZE_U64];
+ LZ4_stream_t_internal internal_donotuse;
+} LZ4_stream_t;
+
+/*
+ * LZ4_streamHC_t - information structure to track an LZ4HC stream.
+ */
+typedef struct {
+ unsigned int hashTable[LZ4HC_HASHTABLESIZE];
+ unsigned short chainTable[LZ4HC_MAXD];
+ /* next block to continue on current prefix */
+ const unsigned char *end;
+ /* All index relative to this position */
+ const unsigned char *base;
+ /* alternate base for extDict */
+ const unsigned char *dictBase;
+ /* below that point, need extDict */
+ unsigned int dictLimit;
+ /* below that point, no more dict */
+ unsigned int lowLimit;
+ /* index from which to continue dict update */
+ unsigned int nextToUpdate;
+ unsigned int compressionLevel;
+} LZ4HC_CCtx_internal;
+typedef union {
+ size_t table[LZ4_STREAMHCSIZE_SIZET];
+ LZ4HC_CCtx_internal internal_donotuse;
+} LZ4_streamHC_t;
+
+/*
+ * LZ4_streamDecode_t - information structure to track an
+ * LZ4 stream during decompression.
+ *
+ * init this structure using LZ4_setStreamDecode (or memset()) before first use
+ */
+typedef struct {
+ const uint8_t *externalDict;
+ size_t extDictSize;
+ const uint8_t *prefixEnd;
+ size_t prefixSize;
+} LZ4_streamDecode_t_internal;
+typedef union {
+ unsigned long long table[LZ4_STREAMDECODESIZE_U64];
+ LZ4_streamDecode_t_internal internal_donotuse;
+} LZ4_streamDecode_t;
+
+/*-************************************************************************
+ * SIZE OF STATE
+ **************************************************************************/
+#define LZ4_MEM_COMPRESS LZ4_STREAMSIZE
+#define LZ4HC_MEM_COMPRESS LZ4_STREAMHCSIZE
+
+/*-************************************************************************
+ * Compression Functions
+ **************************************************************************/
+
+/**
+ * LZ4_compressBound() - Max. output size in worst case szenarios
+ * @isize: Size of the input data
+ *
+ * Return: Max. size LZ4 may output in a "worst case" szenario
+ * (data not compressible)
+ */
+static inline int LZ4_compressBound(size_t isize)
{
- return isize + (isize / 255) + 16;
+ return LZ4_COMPRESSBOUND(isize);
}
-/*
- * lz4_compress()
- * src : source address of the original data
- * src_len : size of the original data
- * dst : output buffer address of the compressed data
- * This requires 'dst' of size LZ4_COMPRESSBOUND.
- * dst_len : is the output size, which is returned after compress done
- * workmem : address of the working memory.
- * This requires 'workmem' of size LZ4_MEM_COMPRESS.
- * return : Success if return 0
- * Error if return (< 0)
- * note : Destination buffer and workmem must be already allocated with
- * the defined size.
- */
-int lz4_compress(const unsigned char *src, size_t src_len,
- unsigned char *dst, size_t *dst_len, void *wrkmem);
-
- /*
- * lz4hc_compress()
- * src : source address of the original data
- * src_len : size of the original data
- * dst : output buffer address of the compressed data
- * This requires 'dst' of size LZ4_COMPRESSBOUND.
- * dst_len : is the output size, which is returned after compress done
- * workmem : address of the working memory.
- * This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
- * return : Success if return 0
- * Error if return (< 0)
- * note : Destination buffer and workmem must be already allocated with
- * the defined size.
- */
-int lz4hc_compress(const unsigned char *src, size_t src_len,
- unsigned char *dst, size_t *dst_len, void *wrkmem);
+/**
+ * lz4_compressbound() - For backwards compatibility; see LZ4_compressBound
+ * @isize: Size of the input data
+ *
+ * Return: Max. size LZ4 may output in a "worst case" szenario
+ * (data not compressible)
+ */
+static inline int lz4_compressbound(size_t isize)
+{
+ return LZ4_COMPRESSBOUND(isize);
+}
+
+/**
+ * LZ4_compress_default() - Compress data from source to dest
+ * @source: source address of the original data
+ * @dest: output buffer address of the compressed data
+ * @inputSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE
+ * @maxOutputSize: full or partial size of buffer 'dest'
+ * which must be already allocated
+ * @wrkmem: address of the working memory.
+ * This requires 'workmem' of LZ4_MEM_COMPRESS.
+ *
+ * Compresses 'sourceSize' bytes from buffer 'source'
+ * into already allocated 'dest' buffer of size 'maxOutputSize'.
+ * Compression is guaranteed to succeed if
+ * 'maxOutputSize' >= LZ4_compressBound(inputSize).
+ * It also runs faster, so it's a recommended setting.
+ * If the function cannot compress 'source' into a more limited 'dest' budget,
+ * compression stops *immediately*, and the function result is zero.
+ * As a consequence, 'dest' content is not valid.
+ *
+ * Return: Number of bytes written into buffer 'dest'
+ * (necessarily <= maxOutputSize) or 0 if compression fails
+ */
+int LZ4_compress_default(const char *source, char *dest, int inputSize,
+ int maxOutputSize, void *wrkmem);
+
+/**
+ * LZ4_compress_fast() - As LZ4_compress_default providing an acceleration param
+ * @source: source address of the original data
+ * @dest: output buffer address of the compressed data
+ * @inputSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE
+ * @maxOutputSize: full or partial size of buffer 'dest'
+ * which must be already allocated
+ * @acceleration: acceleration factor
+ * @wrkmem: address of the working memory.
+ * This requires 'workmem' of LZ4_MEM_COMPRESS.
+ *
+ * Same as LZ4_compress_default(), but allows to select an "acceleration"
+ * factor. The larger the acceleration value, the faster the algorithm,
+ * but also the lesser the compression. It's a trade-off. It can be fine tuned,
+ * with each successive value providing roughly +~3% to speed.
+ * An acceleration value of "1" is the same as regular LZ4_compress_default()
+ * Values <= 0 will be replaced by LZ4_ACCELERATION_DEFAULT, which is 1.
+ *
+ * Return: Number of bytes written into buffer 'dest'
+ * (necessarily <= maxOutputSize) or 0 if compression fails
+ */
+int LZ4_compress_fast(const char *source, char *dest, int inputSize,
+ int maxOutputSize, int acceleration, void *wrkmem);
+
+/**
+ * LZ4_compress_destSize() - Compress as much data as possible
+ * from source to dest
+ * @source: source address of the original data
+ * @dest: output buffer address of the compressed data
+ * @sourceSizePtr: will be modified to indicate how many bytes where read
+ * from 'source' to fill 'dest'. New value is necessarily <= old value.
+ * @targetDestSize: Size of buffer 'dest' which must be already allocated
+ * @wrkmem: address of the working memory.
+ * This requires 'workmem' of LZ4_MEM_COMPRESS.
+ *
+ * Reverse the logic, by compressing as much data as possible
+ * from 'source' buffer into already allocated buffer 'dest'
+ * of size 'targetDestSize'.
+ * This function either compresses the entire 'source' content into 'dest'
+ * if it's large enough, or fill 'dest' buffer completely with as much data as
+ * possible from 'source'.
+ *
+ * Return: Number of bytes written into 'dest' (necessarily <= targetDestSize)
+ * or 0 if compression fails
+ */
+int LZ4_compress_destSize(const char *source, char *dest, int *sourceSizePtr,
+ int targetDestSize, void *wrkmem);
/*
- * lz4_decompress()
- * src : source address of the compressed data
- * src_len : is the input size, whcih is returned after decompress done
- * dest : output buffer address of the decompressed data
- * actual_dest_len: is the size of uncompressed data, supposing it's known
- * return : Success if return 0
- * Error if return (< 0)
- * note : Destination buffer must be already allocated.
- * slightly faster than lz4_decompress_unknownoutputsize()
+ * lz4_compress() - For backward compatibility, see LZ4_compress_default
+ * @src: source address of the original data
+ * @src_len: size of the original data
+ * @dst: output buffer address of the compressed data. This requires 'dst'
+ * of size LZ4_COMPRESSBOUND
+ * @dst_len: is the output size, which is returned after compress done
+ * @workmem: address of the working memory.
+ *
+ * Return: Success if return 0, Error if return < 0
*/
-int lz4_decompress(const unsigned char *src, size_t *src_len,
- unsigned char *dest, size_t actual_dest_len);
+int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
+ size_t *dst_len, void *wrkmem);
+
+/*-************************************************************************
+ * Decompression Functions
+ **************************************************************************/
+
+/**
+ * LZ4_decompress_fast() - Decompresses data from 'source' into 'dest'
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the uncompressed data
+ * which must be already allocated with 'originalSize' bytes
+ * @originalSize: is the original and therefore uncompressed size
+ *
+ * Decompresses data from 'source' into 'dest'.
+ * This function fully respect memory boundaries for properly formed
+ * compressed data.
+ * It is a bit faster than LZ4_decompress_safe().
+ * However, it does not provide any protection against intentionally
+ * modified data stream (malicious input).
+ * Use this function in trusted environment only
+ * (data to decode comes from a trusted source).
+ *
+ * Return: number of bytes read from the source buffer
+ * or a negative result if decompression fails.
+ */
+int LZ4_decompress_fast(const char *source, char *dest, int originalSize);
+
+/**
+ * LZ4_decompress_safe() - Decompression protected against buffer overflow
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the uncompressed data
+ * which must be already allocated
+ * @compressedSize: is the precise full size of the compressed block
+ * @maxDecompressedSize: is the size of 'dest' buffer
+ *
+ * Decompresses data fom 'source' into 'dest'.
+ * If the source stream is detected malformed, the function will
+ * stop decoding and return a negative result.
+ * This function is protected against buffer overflow exploits,
+ * including malicious data packets. It never writes outside output buffer,
+ * nor reads outside input buffer.
+ *
+ * Return: number of bytes decompressed into destination buffer
+ * (necessarily <= maxDecompressedSize)
+ * or a negative result in case of error
+ */
+int LZ4_decompress_safe(const char *source, char *dest, int compressedSize,
+ int maxDecompressedSize);
+
+/**
+ * LZ4_decompress_safe_partial() - Decompress a block of size 'compressedSize'
+ * at position 'source' into buffer 'dest'
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the decompressed data which must be
+ * already allocated
+ * @compressedSize: is the precise full size of the compressed block.
+ * @targetOutputSize: the decompression operation will try
+ * to stop as soon as 'targetOutputSize' has been reached
+ * @maxDecompressedSize: is the size of destination buffer
+ *
+ * This function decompresses a compressed block of size 'compressedSize'
+ * at position 'source' into destination buffer 'dest'
+ * of size 'maxDecompressedSize'.
+ * The function tries to stop decompressing operation as soon as
+ * 'targetOutputSize' has been reached, reducing decompression time.
+ * This function never writes outside of output buffer,
+ * and never reads outside of input buffer.
+ * It is therefore protected against malicious data packets.
+ *
+ * Return: the number of bytes decoded in the destination buffer
+ * (necessarily <= maxDecompressedSize)
+ * or a negative result in case of error
+ *
+ */
+int LZ4_decompress_safe_partial(const char *source, char *dest,
+ int compressedSize, int targetOutputSize, int maxDecompressedSize);
/*
- * lz4_decompress_unknownoutputsize()
- * src : source address of the compressed data
- * src_len : is the input size, therefore the compressed size
- * dest : output buffer address of the decompressed data
- * dest_len: is the max size of the destination buffer, which is
- * returned with actual size of decompressed data after
- * decompress done
- * return : Success if return 0
- * Error if return (< 0)
- * note : Destination buffer must be already allocated.
+ * lz4_decompress_unknownoutputsize() - For backwards compatibility,
+ * see LZ4_decompress_safe
+ * @src: source address of the compressed data
+ * @src_len: is the input size, therefore the compressed size
+ * @dest: output buffer address of the decompressed data
+ * which must be already allocated
+ * @dest_len: is the max size of the destination buffer, which is
+ * returned with actual size of decompressed data after decompress done
+ *
+ * Return: Success if return 0, Error if return (< 0)
*/
int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
- unsigned char *dest, size_t *dest_len);
+ unsigned char *dest, size_t *dest_len);
+
+/**
+ * lz4_decompress() - For backwards cocmpatibility, see LZ4_decompress_fast
+ * @src: source address of the compressed data
+ * @src_len: is the input size, which is returned after decompress done
+ * @dest: output buffer address of the decompressed data,
+ * which must be already allocated
+ * @actual_dest_len: is the size of uncompressed data, supposing it's known
+ *
+ * Return: Success if return 0, Error if return (< 0)
+ */
+int lz4_decompress(const unsigned char *src, size_t *src_len,
+ unsigned char *dest, size_t actual_dest_len);
+
+/*-************************************************************************
+ * LZ4 HC Compression
+ **************************************************************************/
+
+/**
+ * LZ4_compress_HC() - Compress data from `src` into `dst`, using HC algorithm
+ * @src: source address of the original data
+ * @dst: output buffer address of the compressed data
+ * @srcSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE
+ * @dstCapacity: full or partial size of buffer 'dst',
+ * which must be already allocated
+ * @compressionLevel: Recommended values are between 4 and 9, although any
+ * value between 1 and LZ4HC_MAX_CLEVEL will work.
+ * Values >LZ4HC_MAX_CLEVEL behave the same as 16.
+ * @wrkmem: address of the working memory.
+ * This requires 'wrkmem' of size LZ4HC_MEM_COMPRESS.
+ *
+ * Compress data from 'src' into 'dst', using the more powerful
+ * but slower "HC" algorithm. Compression is guaranteed to succeed if
+ * `dstCapacity >= LZ4_compressBound(srcSize)
+ *
+ * Return : the number of bytes written into 'dst' or 0 if compression fails.
+ */
+int LZ4_compress_HC(const char *src, char *dst, int srcSize, int dstCapacity,
+ int compressionLevel, void *wrkmem);
+
+/**
+ * lz4hc_compress() - For backwards compatibility, see LZ4_compress_HC
+ * @src: source address of the original data
+ * @src_len: size of the original data
+ * @dst: output buffer address of the compressed data. This requires 'dst'
+ * of size LZ4_COMPRESSBOUND.
+ * @dst_len: is the output size, which is returned after compress done
+ * @wrkmem: address of the working memory.
+ * This requires 'workmem' of size LZ4HC_MEM_COMPRESS.
+ *
+ * Return : Success if return 0, Error if return (< 0)
+ */
+int lz4hc_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
+ size_t *dst_len, void *wrkmem);
+
+/**
+ * LZ4_resetStreamHC() - Init an allocated 'LZ4_streamHC_t' structure
+ * @streamHCPtr: pointer to the 'LZ4_streamHC_t' structure
+ * @compressionLevel: Recommended values are between 4 and 9, although any
+ * value between 1 and LZ4HC_MAX_CLEVEL will work.
+ * Values >LZ4HC_MAX_CLEVEL behave the same as 16.
+ *
+ * An LZ4_streamHC_t structure can be allocated once
+ * and re-used multiple times.
+ * Use this function to init an allocated `LZ4_streamHC_t` structure
+ * and start a new compression.
+ */
+void LZ4_resetStreamHC(LZ4_streamHC_t *streamHCPtr, int compressionLevel);
+
+/**
+ * LZ4_loadDictHC() - Load a static dictionary into LZ4_streamHC
+ * @streamHCPtr: pointer to the LZ4HC_stream_t
+ * @dictionary: dictionary to load
+ * @dictSize: size of dictionary
+ *
+ * Use this function to load a static dictionary into LZ4HC_stream.
+ * Any previous data will be forgotten, only 'dictionary'
+ * will remain in memory.
+ * Loading a size of 0 is allowed.
+ *
+ * Return : dictionary size, in bytes (necessarily <= 64 KB)
+ */
+int LZ4_loadDictHC(LZ4_streamHC_t *streamHCPtr, const char *dictionary,
+ int dictSize);
+
+/**
+ * LZ4_compress_HC_continue() - Compress 'src' using data from previously
+ * compressed blocks as a dictionary using the HC algorithm
+ * @streamHCPtr: Pointer to the previous 'LZ4_streamHC_t' structure
+ * @src: source address of the original data
+ * @dst: output buffer address of the compressed data,
+ * which must be already allocated
+ * @srcSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE
+ * @maxDstSize: full or partial size of buffer 'dest'
+ * which must be already allocated
+ *
+ * These functions compress data in successive blocks of any size, using
+ * previous blocks as dictionary. One key assumption is that previous
+ * blocks (up to 64 KB) remain read-accessible while
+ * compressing next blocks. There is an exception for ring buffers,
+ * which can be smaller than 64 KB.
+ * Ring buffers scenario is automatically detected and handled by
+ * LZ4_compress_HC_continue().
+ * Before starting compression, state must be properly initialized,
+ * using LZ4_resetStreamHC().
+ * A first "fictional block" can then be designated as
+ * initial dictionary, using LZ4_loadDictHC() (Optional).
+ * Then, use LZ4_compress_HC_continue()
+ * to compress each successive block. Previous memory blocks
+ * (including initial dictionary when present) must remain accessible
+ * and unmodified during compression.
+ * 'dst' buffer should be sized to handle worst case scenarios, using
+ * LZ4_compressBound(), to ensure operation success.
+ * If, for any reason, previous data blocks can't be preserved unmodified
+ * in memory during next compression block,
+ * you must save it to a safer memory space, using LZ4_saveDictHC().
+ * Return value of LZ4_saveDictHC() is the size of dictionary
+ * effectively saved into 'safeBuffer'.
+ *
+ * Return: Number of bytes written into buffer 'dst' or 0 if compression fails
+ */
+int LZ4_compress_HC_continue(LZ4_streamHC_t *streamHCPtr, const char *src,
+ char *dst, int srcSize, int maxDstSize);
+
+/**
+ * LZ4_saveDictHC() - Save static dictionary from LZ4HC_stream
+ * @streamHCPtr: pointer to the 'LZ4HC_stream_t' structure
+ * @safeBuffer: buffer to save dictionary to, must be already allocated
+ * @maxDictSize: size of 'safeBuffer'
+ *
+ * If previously compressed data block is not guaranteed
+ * to remain available at its memory location,
+ * save it into a safer place (char *safeBuffer).
+ * Note : you don't need to call LZ4_loadDictHC() afterwards,
+ * dictionary is immediately usable, you can therefore call
+ * LZ4_compress_HC_continue().
+ *
+ * Return : saved dictionary size in bytes (necessarily <= maxDictSize),
+ * or 0 if error.
+ */
+int LZ4_saveDictHC(LZ4_streamHC_t *streamHCPtr, char *safeBuffer,
+ int maxDictSize);
+
+/*-*********************************************
+ * Streaming Compression Functions
+ ***********************************************/
+
+/**
+ * LZ4_resetStream() - Init an allocated 'LZ4_stream_t' structure
+ * @LZ4_stream: pointer to the 'LZ4_stream_t' structure
+ *
+ * An LZ4_stream_t structure can be allocated once
+ * and re-used multiple times.
+ * Use this function to init an allocated `LZ4_stream_t` structure
+ * and start a new compression.
+ */
+void LZ4_resetStream(LZ4_stream_t *LZ4_stream);
+
+/**
+ * LZ4_loadDict() - Load a static dictionary into LZ4_stream
+ * @streamPtr: pointer to the LZ4_stream_t
+ * @dictionary: dictionary to load
+ * @dictSize: size of dictionary
+ *
+ * Use this function to load a static dictionary into LZ4_stream.
+ * Any previous data will be forgotten, only 'dictionary'
+ * will remain in memory.
+ * Loading a size of 0 is allowed.
+ *
+ * Return : dictionary size, in bytes (necessarily <= 64 KB)
+ */
+int LZ4_loadDict(LZ4_stream_t *streamPtr, const char *dictionary,
+ int dictSize);
+
+/**
+ * LZ4_saveDict() - Save static dictionary from LZ4_stream
+ * @streamPtr: pointer to the 'LZ4_stream_t' structure
+ * @safeBuffer: buffer to save dictionary to, must be already allocated
+ * @dictSize: size of 'safeBuffer'
+ *
+ * If previously compressed data block is not guaranteed
+ * to remain available at its memory location,
+ * save it into a safer place (char *safeBuffer).
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ * dictionary is immediately usable, you can therefore call
+ * LZ4_compress_fast_continue().
+ *
+ * Return : saved dictionary size in bytes (necessarily <= dictSize),
+ * or 0 if error.
+ */
+int LZ4_saveDict(LZ4_stream_t *streamPtr, char *safeBuffer, int dictSize);
+
+/**
+ * LZ4_compress_fast_continue() - Compress 'src' using data from previously
+ * compressed blocks as a dictionary
+ * @streamPtr: Pointer to the previous 'LZ4_stream_t' structure
+ * @src: source address of the original data
+ * @dst: output buffer address of the compressed data,
+ * which must be already allocated
+ * @srcSize: size of the input data. Max supported value is LZ4_MAX_INPUT_SIZE
+ * @maxDstSize: full or partial size of buffer 'dest'
+ * which must be already allocated
+ * @acceleration: acceleration factor
+ *
+ * Compress buffer content 'src', using data from previously compressed blocks
+ * as dictionary to improve compression ratio.
+ * Important : Previous data blocks are assumed to still
+ * be present and unmodified !
+ * If maxDstSize >= LZ4_compressBound(srcSize),
+ * compression is guaranteed to succeed, and runs faster.
+ *
+ * Return: Number of bytes written into buffer 'dst' or 0 if compression fails
+ */
+int LZ4_compress_fast_continue(LZ4_stream_t *streamPtr, const char *src,
+ char *dst, int srcSize, int maxDstSize, int acceleration);
+
+/**
+ * LZ4_setStreamDecode() - Instruct where to find dictionary
+ * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure
+ * @dictionary: dictionary to use
+ * @dictSize: size of dictionary
+ *
+ * Use this function to instruct where to find the dictionary.
+ * Setting a size of 0 is allowed (same effect as reset).
+ *
+ * Return: 1 if OK, 0 if error
+ */
+int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode,
+ const char *dictionary, int dictSize);
+
+/**
+ * LZ4_decompress_fast_continue() - Decompress blocks in streaming mode
+ * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the uncompressed data
+ * which must be already allocated
+ * @compressedSize: is the precise full size of the compressed block
+ * @maxDecompressedSize: is the size of 'dest' buffer
+ *
+ * These decoding function allows decompression of multiple blocks
+ * in "streaming" mode.
+ * Previously decoded blocks *must* remain available at the memory position
+ * where they were decoded (up to 64 KB)
+ * In the case of a ring buffers, decoding buffer must be either :
+ * - Exactly same size as encoding buffer, with same update rule
+ * (block boundaries at same positions) In which case,
+ * the decoding & encoding ring buffer can have any size,
+ * including very small ones ( < 64 KB).
+ * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ * maxBlockSize is implementation dependent.
+ * It's the maximum size you intend to compress into a single block.
+ * In which case, encoding and decoding buffers do not need
+ * to be synchronized, and encoding ring buffer can have any size,
+ * including small ones ( < 64 KB).
+ * - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ * In which case, encoding and decoding buffers do not need to be
+ * synchronized, and encoding ring buffer can have any size,
+ * including larger than decoding buffer. W
+ * Whenever these conditions are not possible, save the last 64KB of decoded
+ * data into a safe buffer, and indicate where it is saved
+ * using LZ4_setStreamDecode()
+ *
+ * Return: number of bytes decompressed into destination buffer
+ * (necessarily <= maxDecompressedSize)
+ * or a negative result in case of error
+ */
+int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode,
+ const char *source, char *dest, int compressedSize,
+ int maxDecompressedSize);
+
+/**
+ * LZ4_decompress_fast_continue() - Decompress blocks in streaming mode
+ * @LZ4_streamDecode: the 'LZ4_streamDecode_t' structure
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the uncompressed data
+ * which must be already allocated with 'originalSize' bytes
+ * @originalSize: is the original and therefore uncompressed size
+ *
+ * These decoding function allows decompression of multiple blocks
+ * in "streaming" mode.
+ * Previously decoded blocks *must* remain available at the memory position
+ * where they were decoded (up to 64 KB)
+ * In the case of a ring buffers, decoding buffer must be either :
+ * - Exactly same size as encoding buffer, with same update rule
+ * (block boundaries at same positions) In which case,
+ * the decoding & encoding ring buffer can have any size,
+ * including very small ones ( < 64 KB).
+ * - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+ * maxBlockSize is implementation dependent.
+ * It's the maximum size you intend to compress into a single block.
+ * In which case, encoding and decoding buffers do not need
+ * to be synchronized, and encoding ring buffer can have any size,
+ * including small ones ( < 64 KB).
+ * - _At least_ 64 KB + 8 bytes + maxBlockSize.
+ * In which case, encoding and decoding buffers do not need to be
+ * synchronized, and encoding ring buffer can have any size,
+ * including larger than decoding buffer. W
+ * Whenever these conditions are not possible, save the last 64KB of decoded
+ * data into a safe buffer, and indicate where it is saved
+ * using LZ4_setStreamDecode()
+ *
+ * Return: number of bytes decompressed into destination buffer
+ * (necessarily <= maxDecompressedSize)
+ * or a negative result in case of error
+ */
+int LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode,
+ const char *source, char *dest, int originalSize);
+
+/**
+ * LZ4_decompress_safe_usingDict() - Same as LZ4_setStreamDecode()
+ * followed by LZ4_decompress_safe_continue()
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the uncompressed data
+ * which must be already allocated
+ * @compressedSize: is the precise full size of the compressed block
+ * @maxDecompressedSize: is the size of 'dest' buffer
+ * @dictStart: pointer to the start of the dictionary in memory
+ * @dictSize: size of dictionary
+ *
+ * These decoding function works the same as
+ * a combination of LZ4_setStreamDecode() followed by
+ * LZ4_decompress_safe_continue()
+ * It is stand-alone, and don'tn eed a LZ4_streamDecode_t structure.
+ *
+ * Return: number of bytes decompressed into destination buffer
+ * (necessarily <= maxDecompressedSize)
+ * or a negative result in case of error
+ */
+int LZ4_decompress_safe_usingDict(const char *source, char *dest,
+ int compressedSize, int maxDecompressedSize, const char *dictStart,
+ int dictSize);
+
+/**
+ * LZ4_decompress_fast_usingDict() - Same as LZ4_setStreamDecode()
+ * followed by LZ4_decompress_fast_continue()
+ * @source: source address of the compressed data
+ * @dest: output buffer address of the uncompressed data
+ * which must be already allocated with 'originalSize' bytes
+ * @originalSize: is the original and therefore uncompressed size
+ * @dictStart: pointer to the start of the dictionary in memory
+ * @dictSize: size of dictionary
+ *
+ * These decoding function works the same as
+ * a combination of LZ4_setStreamDecode() followed by
+ * LZ4_decompress_safe_continue()
+ * It is stand-alone, and don'tn eed a LZ4_streamDecode_t structure.
+ *
+ * Return: number of bytes decompressed into destination buffer
+ * (necessarily <= maxDecompressedSize)
+ * or a negative result in case of error
+ */
+int LZ4_decompress_fast_usingDict(const char *source, char *dest,
+ int originalSize, const char *dictStart, int dictSize);
+
#endif
diff --git a/lib/lz4/lz4_compress.c b/lib/lz4/lz4_compress.c
index 28321d8..6aa7ac3 100644
--- a/lib/lz4/lz4_compress.c
+++ b/lib/lz4/lz4_compress.c
@@ -1,19 +1,16 @@
/*
* LZ4 - Fast LZ compression algorithm
- * Copyright (C) 2011-2012, Yann Collet.
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
-
+ * Copyright (C) 2011 - 2016, Yann Collet.
+ * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php)
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
- *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -25,417 +22,886 @@
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
* You can contact the author at :
- * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- * - LZ4 source repository : http://code.google.com/p/lz4/
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
*
- * Changed for kernel use by:
- * Chanho Min <chanho.min@lge.com>
+ * Changed for kernel usage by:
+ * Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
*/
+/*-************************************
+ * Dependencies
+ **************************************/
+#include <linux/lz4.h>
+#include "lz4defs.h"
#include <linux/module.h>
#include <linux/kernel.h>
-#include <linux/lz4.h>
#include <asm/unaligned.h>
-#include "lz4defs.h"
-/*
- * LZ4_compressCtx :
- * -----------------
- * Compress 'isize' bytes from 'source' into an output buffer 'dest' of
- * maximum size 'maxOutputSize'. * If it cannot achieve it, compression
- * will stop, and result of the function will be zero.
- * return : the number of bytes written in buffer 'dest', or 0 if the
- * compression fails
- */
-static inline int lz4_compressctx(void *ctx,
- const char *source,
- char *dest,
- int isize,
- int maxoutputsize)
+/*-******************************
+ * Compression functions
+ ********************************/
+static U32 LZ4_hash4(U32 sequence, tableType_t const tableType)
{
- HTYPE *hashtable = (HTYPE *)ctx;
- const u8 *ip = (u8 *)source;
+ if (tableType == byU16)
+ return ((sequence * 2654435761U)
+ >> ((MINMATCH*8) - (LZ4_HASHLOG + 1)));
+ else
+ return ((sequence * 2654435761U)
+ >> ((MINMATCH*8) - LZ4_HASHLOG));
+}
+
#if LZ4_ARCH64
- const BYTE * const base = ip;
+static U32 LZ4_hash5(U64 sequence, tableType_t const tableType)
+{
+ const U32 hashLog = (tableType == byU16)
+ ? LZ4_HASHLOG + 1
+ : LZ4_HASHLOG;
+
+#ifdef __LITTLE_ENDIAN__
+ static const U64 prime5bytes = 889523592379ULL;
+
+ return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog));
#else
- const int base = 0;
+ static const U64 prime8bytes = 11400714785074694791ULL;
+
+ return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog));
#endif
- const u8 *anchor = ip;
- const u8 *const iend = ip + isize;
- const u8 *const mflimit = iend - MFLIMIT;
- #define MATCHLIMIT (iend - LASTLITERALS)
-
- u8 *op = (u8 *) dest;
- u8 *const oend = op + maxoutputsize;
- int length;
- const int skipstrength = SKIPSTRENGTH;
- u32 forwardh;
- int lastrun;
-
- /* Init */
- if (isize < MINLENGTH)
- goto _last_literals;
+}
+#endif
+
+static U32 LZ4_hashPosition(const void *p, tableType_t tableType)
+{
+#if LZ4_ARCH64
+ if (tableType == byU32)
+ return LZ4_hash5(LZ4_read_ARCH(p), tableType);
+#endif
+
+ return LZ4_hash4(LZ4_read32(p), tableType);
+}
+
+static void LZ4_putPositionOnHash(const BYTE *p, U32 h, void *tableBase,
+ tableType_t const tableType, const BYTE *srcBase)
+{
+ switch (tableType) {
+ case byPtr:
+ {
+ const BYTE **hashTable = (const BYTE **)tableBase;
+
+ hashTable[h] = p;
+ return;
+ }
+ case byU32:
+ {
+ U32 *hashTable = (U32 *) tableBase;
+
+ hashTable[h] = (U32)(p - srcBase);
+ return;
+ }
+ case byU16:
+ {
+ U16 *hashTable = (U16 *) tableBase;
+
+ hashTable[h] = (U16)(p - srcBase);
+ return;
+ }
+ }
+}
+
+static inline void LZ4_putPosition(const BYTE *p, void *tableBase,
+ tableType_t tableType, const BYTE *srcBase)
+{
+ U32 const h = LZ4_hashPosition(p, tableType);
+
+ LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase);
+}
+
+static const BYTE *LZ4_getPositionOnHash(U32 h, void *tableBase,
+ tableType_t tableType, const BYTE *srcBase)
+{
+ if (tableType == byPtr) {
+ const BYTE **hashTable = (const BYTE **) tableBase;
+
+ return hashTable[h];
+ }
+
+ if (tableType == byU32) {
+ const U32 * const hashTable = (U32 *) tableBase;
+
+ return hashTable[h] + srcBase;
+ }
+
+ {
+ /* default, to ensure a return */
+ const U16 * const hashTable = (U16 *) tableBase;
+ return hashTable[h] + srcBase;
+ }
+}
+
+static inline const BYTE *LZ4_getPosition(const BYTE *p, void *tableBase,
+ tableType_t tableType, const BYTE *srcBase)
+{
+ U32 const h = LZ4_hashPosition(p, tableType);
+
+ return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase);
+}
+
+
+/*
+ * LZ4_compress_generic() :
+ * inlined, to ensure branches are decided at compilation time
+ */
+static inline int LZ4_compress_generic(
+ LZ4_stream_t_internal * const dictPtr,
+ const char * const source,
+ char * const dest,
+ const int inputSize,
+ const int maxOutputSize,
+ const limitedOutput_directive outputLimited,
+ const tableType_t tableType,
+ const dict_directive dict,
+ const dictIssue_directive dictIssue,
+ const U32 acceleration)
+{
+ const BYTE *ip = (const BYTE *) source;
+ const BYTE *base;
+ const BYTE *lowLimit;
+ const BYTE * const lowRefLimit = ip - dictPtr->dictSize;
+ const BYTE * const dictionary = dictPtr->dictionary;
+ const BYTE * const dictEnd = dictionary + dictPtr->dictSize;
+ const size_t dictDelta = dictEnd - (const BYTE *)source;
+ const BYTE *anchor = (const BYTE *) source;
+ const BYTE * const iend = ip + inputSize;
+ const BYTE * const mflimit = iend - MFLIMIT;
+ const BYTE * const matchlimit = iend - LASTLITERALS;
+
+ BYTE *op = (BYTE *) dest;
+ BYTE * const olimit = op + maxOutputSize;
+
+ U32 forwardH;
+ size_t refDelta = 0;
+
+ /* Init conditions */
+ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) {
+ /* Unsupported inputSize, too large (or negative) */
+ return 0;
+ }
+ switch (dict) {
+ case noDict:
+ default:
+ base = (const BYTE *)source;
+ lowLimit = (const BYTE *)source;
+ break;
+ case withPrefix64k:
+ base = (const BYTE *)source - dictPtr->currentOffset;
+ lowLimit = (const BYTE *)source - dictPtr->dictSize;
+ break;
+ case usingExtDict:
+ base = (const BYTE *)source - dictPtr->currentOffset;
+ lowLimit = (const BYTE *)source;
+ break;
+ }
+
+ if ((tableType == byU16)
+ && (inputSize >= LZ4_64Klimit)) {
+ /* Size too large (not within 64K limit) */
+ return 0;
+ }
- memset((void *)hashtable, 0, LZ4_MEM_COMPRESS);
+ if (inputSize < LZ4_minLength) {
+ /* Input too small, no compression (all literals) */
+ goto _last_literals;
+ }
/* First Byte */
- hashtable[LZ4_HASH_VALUE(ip)] = ip - base;
- ip++;
- forwardh = LZ4_HASH_VALUE(ip);
+ LZ4_putPosition(ip, dictPtr->hashTable, tableType, base);
+ ip++; forwardH = LZ4_hashPosition(ip, tableType);
/* Main Loop */
- for (;;) {
- int findmatchattempts = (1U << skipstrength) + 3;
- const u8 *forwardip = ip;
- const u8 *ref;
- u8 *token;
+ for ( ; ; ) {
+ const BYTE *match;
+ BYTE *token;
/* Find a match */
- do {
- u32 h = forwardh;
- int step = findmatchattempts++ >> skipstrength;
- ip = forwardip;
- forwardip = ip + step;
-
- if (unlikely(forwardip > mflimit))
- goto _last_literals;
-
- forwardh = LZ4_HASH_VALUE(forwardip);
- ref = base + hashtable[h];
- hashtable[h] = ip - base;
- } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip)));
+ {
+ const BYTE *forwardIp = ip;
+ unsigned int step = 1;
+ unsigned int searchMatchNb = acceleration
+ << LZ4_skipTrigger;
+
+ do {
+ U32 const h = forwardH;
+
+ ip = forwardIp;
+ forwardIp += step;
+ step = (searchMatchNb++ >> LZ4_skipTrigger);
+
+ if (unlikely(forwardIp > mflimit))
+ goto _last_literals;
+
+ match = LZ4_getPositionOnHash(h,
+ dictPtr->hashTable,
+ tableType, base);
+ if (dict == usingExtDict) {
+ if (match < (const BYTE *)source) {
+ refDelta = dictDelta;
+ lowLimit = dictionary;
+ } else {
+ refDelta = 0;
+ lowLimit = (const BYTE *)source;
+ } }
+ forwardH = LZ4_hashPosition(forwardIp,
+ tableType);
+ LZ4_putPositionOnHash(ip, h, dictPtr->hashTable,
+ tableType, base);
+
+ } while (((dictIssue == dictSmall)
+ ? (match < lowRefLimit)
+ : 0)
+ || ((tableType == byU16)
+ ? 0
+ : (match + MAX_DISTANCE < ip))
+ || (LZ4_read32(match + refDelta)
+ != LZ4_read32(ip)));
+ }
/* Catch up */
- while ((ip > anchor) && (ref > (u8 *)source) &&
- unlikely(ip[-1] == ref[-1])) {
+ while (((ip > anchor) & (match + refDelta > lowLimit))
+ && (unlikely(ip[-1] == match[refDelta - 1]))) {
ip--;
- ref--;
- }
+ match--;
+ }
- /* Encode Literal length */
- length = (int)(ip - anchor);
- token = op++;
- /* check output limit */
- if (unlikely(op + length + (2 + 1 + LASTLITERALS) +
- (length >> 8) > oend))
- return 0;
+ /* Encode Literals */
+ {
+ unsigned const int litLength = (unsigned int)(ip - anchor);
- if (length >= (int)RUN_MASK) {
- int len;
- *token = (RUN_MASK << ML_BITS);
- len = length - RUN_MASK;
- for (; len > 254 ; len -= 255)
- *op++ = 255;
- *op++ = (u8)len;
- } else
- *token = (length << ML_BITS);
+ token = op++;
+ if ((outputLimited) &&
+ /* Check output buffer overflow */
+ (unlikely(op + litLength +
+ (2 + 1 + LASTLITERALS) +
+ (litLength/255) > olimit)))
+ return 0;
+ if (litLength >= RUN_MASK) {
+ int len = (int)litLength - RUN_MASK;
+
+ *token = (RUN_MASK<<ML_BITS);
+ for (; len >= 255 ; len -= 255)
+ *op++ = 255;
+ *op++ = (BYTE)len;
+ } else
+ *token = (BYTE)(litLength<<ML_BITS);
+
+ /* Copy Literals */
+ LZ4_wildCopy(op, anchor, op + litLength);
+ op += litLength;
+ }
- /* Copy Literals */
- LZ4_BLINDCOPY(anchor, op, length);
_next_match:
/* Encode Offset */
- LZ4_WRITE_LITTLEENDIAN_16(op, (u16)(ip - ref));
+ LZ4_writeLE16(op, (U16)(ip - match)); op += 2;
- /* Start Counting */
- ip += MINMATCH;
- /* MinMatch verified */
- ref += MINMATCH;
- anchor = ip;
- while (likely(ip < MATCHLIMIT - (STEPSIZE - 1))) {
- #if LZ4_ARCH64
- u64 diff = A64(ref) ^ A64(ip);
- #else
- u32 diff = A32(ref) ^ A32(ip);
- #endif
- if (!diff) {
- ip += STEPSIZE;
- ref += STEPSIZE;
- continue;
- }
- ip += LZ4_NBCOMMONBYTES(diff);
- goto _endcount;
- }
- #if LZ4_ARCH64
- if ((ip < (MATCHLIMIT - 3)) && (A32(ref) == A32(ip))) {
- ip += 4;
- ref += 4;
- }
- #endif
- if ((ip < (MATCHLIMIT - 1)) && (A16(ref) == A16(ip))) {
- ip += 2;
- ref += 2;
- }
- if ((ip < MATCHLIMIT) && (*ref == *ip))
- ip++;
-_endcount:
/* Encode MatchLength */
- length = (int)(ip - anchor);
- /* Check output limit */
- if (unlikely(op + (1 + LASTLITERALS) + (length >> 8) > oend))
- return 0;
- if (length >= (int)ML_MASK) {
- *token += ML_MASK;
- length -= ML_MASK;
- for (; length > 509 ; length -= 510) {
- *op++ = 255;
- *op++ = 255;
- }
- if (length > 254) {
- length -= 255;
- *op++ = 255;
+ {
+ unsigned int matchCode;
+
+ if ((dict == usingExtDict)
+ && (lowLimit == dictionary)) {
+ const BYTE *limit;
+
+ match += refDelta;
+ limit = ip + (dictEnd - match);
+ if (limit > matchlimit)
+ limit = matchlimit;
+ matchCode = LZ4_count(ip + MINMATCH,
+ match + MINMATCH, limit);
+ ip += MINMATCH + matchCode;
+ if (ip == limit) {
+ unsigned const int more = LZ4_count(ip,
+ (const BYTE *)source,
+ matchlimit);
+
+ matchCode += more;
+ ip += more;
+ }
+ } else {
+ matchCode = LZ4_count(ip + MINMATCH,
+ match + MINMATCH, matchlimit);
+ ip += MINMATCH + matchCode;
}
- *op++ = (u8)length;
- } else
- *token += length;
+
+ if (outputLimited &&
+ /* Check output buffer overflow */
+ (unlikely(op +
+ (1 + LASTLITERALS) +
+ (matchCode>>8) > olimit)))
+ return 0;
+ if (matchCode >= ML_MASK) {
+ *token += ML_MASK;
+ matchCode -= ML_MASK;
+ LZ4_write32(op, 0xFFFFFFFF);
+ while (matchCode >= 4*255) {
+ op += 4;
+ LZ4_write32(op, 0xFFFFFFFF);
+ matchCode -= 4*255;
+ }
+ op += matchCode / 255;
+ *op++ = (BYTE)(matchCode % 255);
+ } else
+ *token += (BYTE)(matchCode);
+ }
+
+ anchor = ip;
/* Test end of chunk */
- if (ip > mflimit) {
- anchor = ip;
+ if (ip > mflimit)
break;
- }
/* Fill table */
- hashtable[LZ4_HASH_VALUE(ip-2)] = ip - 2 - base;
+ LZ4_putPosition(ip - 2, dictPtr->hashTable, tableType, base);
/* Test next position */
- ref = base + hashtable[LZ4_HASH_VALUE(ip)];
- hashtable[LZ4_HASH_VALUE(ip)] = ip - base;
- if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) {
+ match = LZ4_getPosition(ip, dictPtr->hashTable,
+ tableType, base);
+ if (dict == usingExtDict) {
+ if (match < (const BYTE *)source) {
+ refDelta = dictDelta;
+ lowLimit = dictionary;
+ } else {
+ refDelta = 0;
+ lowLimit = (const BYTE *)source;
+ }
+ }
+ LZ4_putPosition(ip, dictPtr->hashTable, tableType, base);
+ if (((dictIssue == dictSmall) ? (match >= lowRefLimit) : 1)
+ && (match + MAX_DISTANCE >= ip)
+ && (LZ4_read32(match + refDelta) == LZ4_read32(ip))) {
token = op++;
*token = 0;
goto _next_match;
}
/* Prepare next loop */
- anchor = ip++;
- forwardh = LZ4_HASH_VALUE(ip);
+ forwardH = LZ4_hashPosition(++ip, tableType);
}
_last_literals:
/* Encode Last Literals */
- lastrun = (int)(iend - anchor);
- if (((char *)op - dest) + lastrun + 1
- + ((lastrun + 255 - RUN_MASK) / 255) > (u32)maxoutputsize)
- return 0;
+ {
+ size_t const lastRun = (size_t)(iend - anchor);
- if (lastrun >= (int)RUN_MASK) {
- *op++ = (RUN_MASK << ML_BITS);
- lastrun -= RUN_MASK;
- for (; lastrun > 254 ; lastrun -= 255)
- *op++ = 255;
- *op++ = (u8)lastrun;
- } else
- *op++ = (lastrun << ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend - anchor;
+ if ((outputLimited) &&
+ /* Check output buffer overflow */
+ ((op - (BYTE *)dest) + lastRun + 1 +
+ ((lastRun + 255 - RUN_MASK)/255) > (U32)maxOutputSize))
+ return 0;
+ if (lastRun >= RUN_MASK) {
+ size_t accumulator = lastRun - RUN_MASK;
+ *op++ = RUN_MASK << ML_BITS;
+ for (; accumulator >= 255 ; accumulator -= 255)
+ *op++ = 255;
+ *op++ = (BYTE) accumulator;
+ } else {
+ *op++ = (BYTE)(lastRun<<ML_BITS);
+ }
+ memcpy(op, anchor, lastRun);
+ op += lastRun;
+ }
/* End */
- return (int)(((char *)op) - dest);
+ return (int) (((char *)op) - dest);
}
-static inline int lz4_compress64kctx(void *ctx,
- const char *source,
- char *dest,
- int isize,
- int maxoutputsize)
+static int LZ4_compress_fast_extState(void *state, const char *source, char *dest,
+ int inputSize, int maxOutputSize, int acceleration)
{
- u16 *hashtable = (u16 *)ctx;
- const u8 *ip = (u8 *) source;
- const u8 *anchor = ip;
- const u8 *const base = ip;
- const u8 *const iend = ip + isize;
- const u8 *const mflimit = iend - MFLIMIT;
- #define MATCHLIMIT (iend - LASTLITERALS)
-
- u8 *op = (u8 *) dest;
- u8 *const oend = op + maxoutputsize;
- int len, length;
- const int skipstrength = SKIPSTRENGTH;
- u32 forwardh;
- int lastrun;
-
- /* Init */
- if (isize < MINLENGTH)
- goto _last_literals;
+ #if LZ4_ARCH64
+ tableType_t tableType = byU32;
+ #else
+ tableType_t tableType = byPtr;
+ #endif
+
+ LZ4_stream_t_internal *ctx = &((LZ4_stream_t *)state)->internal_donotuse;
+
+ LZ4_resetStream((LZ4_stream_t *)state);
+
+ if (acceleration < 1)
+ acceleration = LZ4_ACCELERATION_DEFAULT;
+
+ if (maxOutputSize >= LZ4_compressBound(inputSize)) {
+ if (inputSize < LZ4_64Klimit)
+ return LZ4_compress_generic(ctx, source,
+ dest, inputSize, 0,
+ noLimit, byU16, noDict,
+ noDictIssue, acceleration);
+ else
+ return LZ4_compress_generic(ctx, source,
+ dest, inputSize, 0,
+ noLimit, tableType, noDict,
+ noDictIssue, acceleration);
+ } else {
+ if (inputSize < LZ4_64Klimit)
+ return LZ4_compress_generic(ctx, source,
+ dest, inputSize,
+ maxOutputSize, limitedOutput, byU16, noDict,
+ noDictIssue, acceleration);
+ else
+ return LZ4_compress_generic(ctx, source,
+ dest, inputSize,
+ maxOutputSize, limitedOutput, tableType, noDict,
+ noDictIssue, acceleration);
+ }
+}
- memset((void *)hashtable, 0, LZ4_MEM_COMPRESS);
+int LZ4_compress_fast(const char *source, char *dest, int inputSize,
+ int maxOutputSize, int acceleration, void *wrkmem)
+{
+ return LZ4_compress_fast_extState(wrkmem, source, dest, inputSize,
+ maxOutputSize, acceleration);
+}
+EXPORT_SYMBOL(LZ4_compress_fast);
+
+int LZ4_compress_default(const char *source, char *dest, int inputSize,
+ int maxOutputSize, void *wrkmem)
+{
+ return LZ4_compress_fast(source, dest, inputSize,
+ maxOutputSize, LZ4_ACCELERATION_DEFAULT, wrkmem);
+}
+EXPORT_SYMBOL(LZ4_compress_default);
+
+/*-******************************
+ * *_destSize() variant
+ ********************************/
+
+static int LZ4_compress_destSize_generic(
+ LZ4_stream_t_internal * const ctx,
+ const char * const src,
+ char * const dst,
+ int * const srcSizePtr,
+ const int targetDstSize,
+ const tableType_t tableType)
+{
+ const BYTE *ip = (const BYTE *) src;
+ const BYTE *base = (const BYTE *) src;
+ const BYTE *lowLimit = (const BYTE *) src;
+ const BYTE *anchor = ip;
+ const BYTE * const iend = ip + *srcSizePtr;
+ const BYTE * const mflimit = iend - MFLIMIT;
+ const BYTE * const matchlimit = iend - LASTLITERALS;
+
+ BYTE *op = (BYTE *) dst;
+ BYTE * const oend = op + targetDstSize;
+ BYTE * const oMaxLit = op + targetDstSize - 2 /* offset */
+ - 8 /* because 8 + MINMATCH == MFLIMIT */ - 1 /* token */;
+ BYTE * const oMaxMatch = op + targetDstSize
+ - (LASTLITERALS + 1 /* token */);
+ BYTE * const oMaxSeq = oMaxLit - 1 /* token */;
+
+ U32 forwardH;
+
+ /* Init conditions */
+ /* Impossible to store anything */
+ if (targetDstSize < 1)
+ return 0;
+ /* Unsupported input size, too large (or negative) */
+ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE)
+ return 0;
+ /* Size too large (not within 64K limit) */
+ if ((tableType == byU16) && (*srcSizePtr >= LZ4_64Klimit))
+ return 0;
+ /* Input too small, no compression (all literals) */
+ if (*srcSizePtr < LZ4_minLength)
+ goto _last_literals;
/* First Byte */
- ip++;
- forwardh = LZ4_HASH64K_VALUE(ip);
+ *srcSizePtr = 0;
+ LZ4_putPosition(ip, ctx->hashTable, tableType, base);
+ ip++; forwardH = LZ4_hashPosition(ip, tableType);
/* Main Loop */
- for (;;) {
- int findmatchattempts = (1U << skipstrength) + 3;
- const u8 *forwardip = ip;
- const u8 *ref;
- u8 *token;
+ for ( ; ; ) {
+ const BYTE *match;
+ BYTE *token;
/* Find a match */
- do {
- u32 h = forwardh;
- int step = findmatchattempts++ >> skipstrength;
- ip = forwardip;
- forwardip = ip + step;
-
- if (forwardip > mflimit)
- goto _last_literals;
-
- forwardh = LZ4_HASH64K_VALUE(forwardip);
- ref = base + hashtable[h];
- hashtable[h] = (u16)(ip - base);
- } while (A32(ref) != A32(ip));
+ {
+ const BYTE *forwardIp = ip;
+ unsigned int step = 1;
+ unsigned int searchMatchNb = 1 << LZ4_skipTrigger;
+
+ do {
+ U32 h = forwardH;
+
+ ip = forwardIp;
+ forwardIp += step;
+ step = (searchMatchNb++ >> LZ4_skipTrigger);
+
+ if (unlikely(forwardIp > mflimit))
+ goto _last_literals;
+
+ match = LZ4_getPositionOnHash(h, ctx->hashTable,
+ tableType, base);
+ forwardH = LZ4_hashPosition(forwardIp,
+ tableType);
+ LZ4_putPositionOnHash(ip, h,
+ ctx->hashTable, tableType,
+ base);
+
+ } while (((tableType == byU16)
+ ? 0
+ : (match + MAX_DISTANCE < ip))
+ || (LZ4_read32(match) != LZ4_read32(ip)));
+ }
/* Catch up */
- while ((ip > anchor) && (ref > (u8 *)source)
- && (ip[-1] == ref[-1])) {
- ip--;
- ref--;
- }
+ while ((ip > anchor)
+ && (match > lowLimit)
+ && (unlikely(ip[-1] == match[-1]))) {
+ ip--; match--;
+ }
/* Encode Literal length */
- length = (int)(ip - anchor);
- token = op++;
- /* Check output limit */
- if (unlikely(op + length + (2 + 1 + LASTLITERALS)
- + (length >> 8) > oend))
- return 0;
- if (length >= (int)RUN_MASK) {
- *token = (RUN_MASK << ML_BITS);
- len = length - RUN_MASK;
- for (; len > 254 ; len -= 255)
- *op++ = 255;
- *op++ = (u8)len;
- } else
- *token = (length << ML_BITS);
+ {
+ unsigned int litLength = (unsigned int)(ip - anchor);
- /* Copy Literals */
- LZ4_BLINDCOPY(anchor, op, length);
+ token = op++;
+ if (op + ((litLength + 240)/255)
+ + litLength > oMaxLit) {
+ /* Not enough space for a last match */
+ op--;
+ goto _last_literals;
+ }
+ if (litLength >= RUN_MASK) {
+ unsigned int len = litLength - RUN_MASK;
+ *token = (RUN_MASK<<ML_BITS);
+ for (; len >= 255 ; len -= 255)
+ *op++ = 255;
+ *op++ = (BYTE)len;
+ } else
+ *token = (BYTE)(litLength<<ML_BITS);
+
+ /* Copy Literals */
+ LZ4_wildCopy(op, anchor, op + litLength);
+ op += litLength;
+ }
_next_match:
/* Encode Offset */
- LZ4_WRITE_LITTLEENDIAN_16(op, (u16)(ip - ref));
+ LZ4_writeLE16(op, (U16)(ip - match)); op += 2;
- /* Start Counting */
- ip += MINMATCH;
- /* MinMatch verified */
- ref += MINMATCH;
- anchor = ip;
+ /* Encode MatchLength */
+ {
+ size_t matchLength = LZ4_count(ip + MINMATCH,
+ match + MINMATCH, matchlimit);
- while (ip < MATCHLIMIT - (STEPSIZE - 1)) {
- #if LZ4_ARCH64
- u64 diff = A64(ref) ^ A64(ip);
- #else
- u32 diff = A32(ref) ^ A32(ip);
- #endif
-
- if (!diff) {
- ip += STEPSIZE;
- ref += STEPSIZE;
- continue;
+ if (op + ((matchLength + 240)/255) > oMaxMatch) {
+ /* Match description too long : reduce it */
+ matchLength = (15 - 1) + (oMaxMatch - op) * 255;
}
- ip += LZ4_NBCOMMONBYTES(diff);
- goto _endcount;
+ ip += MINMATCH + matchLength;
+
+ if (matchLength >= ML_MASK) {
+ *token += ML_MASK;
+ matchLength -= ML_MASK;
+ while (matchLength >= 255) {
+ matchLength -= 255; *op++ = 255;
+ }
+ *op++ = (BYTE)matchLength;
+ } else
+ *token += (BYTE)(matchLength);
}
- #if LZ4_ARCH64
- if ((ip < (MATCHLIMIT - 3)) && (A32(ref) == A32(ip))) {
- ip += 4;
- ref += 4;
- }
- #endif
- if ((ip < (MATCHLIMIT - 1)) && (A16(ref) == A16(ip))) {
- ip += 2;
- ref += 2;
- }
- if ((ip < MATCHLIMIT) && (*ref == *ip))
- ip++;
-_endcount:
- /* Encode MatchLength */
- len = (int)(ip - anchor);
- /* Check output limit */
- if (unlikely(op + (1 + LASTLITERALS) + (len >> 8) > oend))
- return 0;
- if (len >= (int)ML_MASK) {
- *token += ML_MASK;
- len -= ML_MASK;
- for (; len > 509 ; len -= 510) {
- *op++ = 255;
- *op++ = 255;
- }
- if (len > 254) {
- len -= 255;
- *op++ = 255;
- }
- *op++ = (u8)len;
- } else
- *token += len;
+ anchor = ip;
- /* Test end of chunk */
- if (ip > mflimit) {
- anchor = ip;
+ /* Test end of block */
+ if (ip > mflimit)
+ break;
+ if (op > oMaxSeq)
break;
- }
/* Fill table */
- hashtable[LZ4_HASH64K_VALUE(ip-2)] = (u16)(ip - 2 - base);
+ LZ4_putPosition(ip - 2, ctx->hashTable, tableType, base);
/* Test next position */
- ref = base + hashtable[LZ4_HASH64K_VALUE(ip)];
- hashtable[LZ4_HASH64K_VALUE(ip)] = (u16)(ip - base);
- if (A32(ref) == A32(ip)) {
- token = op++;
- *token = 0;
+ match = LZ4_getPosition(ip, ctx->hashTable, tableType, base);
+ LZ4_putPosition(ip, ctx->hashTable, tableType, base);
+
+ if ((match + MAX_DISTANCE >= ip)
+ && (LZ4_read32(match) == LZ4_read32(ip))) {
+ token = op++; *token = 0;
goto _next_match;
}
/* Prepare next loop */
- anchor = ip++;
- forwardh = LZ4_HASH64K_VALUE(ip);
+ forwardH = LZ4_hashPosition(++ip, tableType);
}
_last_literals:
/* Encode Last Literals */
- lastrun = (int)(iend - anchor);
- if (op + lastrun + 1 + (lastrun - RUN_MASK + 255) / 255 > oend)
- return 0;
- if (lastrun >= (int)RUN_MASK) {
- *op++ = (RUN_MASK << ML_BITS);
- lastrun -= RUN_MASK;
- for (; lastrun > 254 ; lastrun -= 255)
- *op++ = 255;
- *op++ = (u8)lastrun;
- } else
- *op++ = (lastrun << ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend - anchor;
+ {
+ size_t lastRunSize = (size_t)(iend - anchor);
+
+ if (op + 1 /* token */
+ + ((lastRunSize + 240)/255) /* litLength */
+ + lastRunSize /* literals */ > oend) {
+ /* adapt lastRunSize to fill 'dst' */
+ lastRunSize = (oend - op) - 1;
+ lastRunSize -= (lastRunSize + 240)/255;
+ }
+ ip = anchor + lastRunSize;
+
+ if (lastRunSize >= RUN_MASK) {
+ size_t accumulator = lastRunSize - RUN_MASK;
+
+ *op++ = RUN_MASK << ML_BITS;
+ for (; accumulator >= 255 ; accumulator -= 255)
+ *op++ = 255;
+ *op++ = (BYTE) accumulator;
+ } else {
+ *op++ = (BYTE)(lastRunSize<<ML_BITS);
+ }
+ memcpy(op, anchor, lastRunSize);
+ op += lastRunSize;
+ }
+
/* End */
- return (int)(((char *)op) - dest);
+ *srcSizePtr = (int) (((const char *)ip) - src);
+ return (int) (((char *)op) - dst);
}
-int lz4_compress(const unsigned char *src, size_t src_len,
- unsigned char *dst, size_t *dst_len, void *wrkmem)
+static int LZ4_compress_destSize_extState(LZ4_stream_t *state, const char *src,
+ char *dst, int *srcSizePtr, int targetDstSize)
{
- int ret = -1;
- int out_len = 0;
+ #if LZ4_ARCH64
+ tableType_t tableType = byU32;
+ #else
+ tableType_t tableType = byPtr;
+ #endif
+
+ LZ4_resetStream(state);
+
+ if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) {
+ /* compression success is guaranteed */
+ return LZ4_compress_fast_extState(
+ state, src, dst, *srcSizePtr,
+ targetDstSize, 1);
+ } else {
+ if (*srcSizePtr < LZ4_64Klimit)
+ return LZ4_compress_destSize_generic(
+ &state->internal_donotuse,
+ src, dst, srcSizePtr,
+ targetDstSize, byU16);
+ else
+ return LZ4_compress_destSize_generic(
+ &state->internal_donotuse,
+ src, dst, srcSizePtr,
+ targetDstSize, tableType);
+ }
+}
- if (src_len < LZ4_64KLIMIT)
- out_len = lz4_compress64kctx(wrkmem, src, dst, src_len,
- lz4_compressbound(src_len));
- else
- out_len = lz4_compressctx(wrkmem, src, dst, src_len,
- lz4_compressbound(src_len));
- if (out_len < 0)
- goto exit;
+int LZ4_compress_destSize(const char *src, char *dst, int *srcSizePtr,
+ int targetDstSize, void *wrkmem)
+{
+ return LZ4_compress_destSize_extState(wrkmem, src, dst, srcSizePtr,
+ targetDstSize);
+}
+EXPORT_SYMBOL(LZ4_compress_destSize);
+
+/*-******************************
+ * Streaming functions
+ ********************************/
+void LZ4_resetStream(LZ4_stream_t *LZ4_stream)
+{
+ memset(LZ4_stream, 0, sizeof(LZ4_stream_t));
+}
+
+#define HASH_UNIT sizeof(size_t)
+int LZ4_loadDict(LZ4_stream_t *LZ4_dict,
+ const char *dictionary, int dictSize)
+{
+ LZ4_stream_t_internal *dict = &LZ4_dict->internal_donotuse;
+ const BYTE *p = (const BYTE *)dictionary;
+ const BYTE * const dictEnd = p + dictSize;
+ const BYTE *base;
+
+ if ((dict->initCheck)
+ || (dict->currentOffset > 1 * GB)) {
+ /* Uninitialized structure, or reuse overflow */
+ LZ4_resetStream(LZ4_dict);
+ }
+
+ if (dictSize < (int)HASH_UNIT) {
+ dict->dictionary = NULL;
+ dict->dictSize = 0;
+ return 0;
+ }
+
+ if ((dictEnd - p) > 64 * KB)
+ p = dictEnd - 64 * KB;
+ dict->currentOffset += 64 * KB;
+ base = p - dict->currentOffset;
+ dict->dictionary = p;
+ dict->dictSize = (U32)(dictEnd - p);
+ dict->currentOffset += dict->dictSize;
+
+ while (p <= dictEnd - HASH_UNIT) {
+ LZ4_putPosition(p, dict->hashTable, byU32, base);
+ p += 3;
+ }
+
+ return dict->dictSize;
+}
+EXPORT_SYMBOL(LZ4_loadDict);
+
+static void LZ4_renormDictT(LZ4_stream_t_internal *LZ4_dict,
+ const BYTE *src)
+{
+ if ((LZ4_dict->currentOffset > 0x80000000) ||
+ ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) {
+ /* address space overflow */
+ /* rescale hash table */
+ U32 const delta = LZ4_dict->currentOffset - 64 * KB;
+ const BYTE *dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize;
+ int i;
+
+ for (i = 0; i < LZ4_HASH_SIZE_U32; i++) {
+ if (LZ4_dict->hashTable[i] < delta)
+ LZ4_dict->hashTable[i] = 0;
+ else
+ LZ4_dict->hashTable[i] -= delta;
+ }
+ LZ4_dict->currentOffset = 64 * KB;
+ if (LZ4_dict->dictSize > 64 * KB)
+ LZ4_dict->dictSize = 64 * KB;
+ LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize;
+ }
+}
+
+int LZ4_saveDict(LZ4_stream_t *LZ4_dict, char *safeBuffer, int dictSize)
+{
+ LZ4_stream_t_internal * const dict = &LZ4_dict->internal_donotuse;
+ const BYTE * const previousDictEnd = dict->dictionary + dict->dictSize;
+
+ if ((U32)dictSize > 64 * KB) {
+ /* useless to define a dictionary > 64 * KB */
+ dictSize = 64 * KB;
+ }
+ if ((U32)dictSize > dict->dictSize)
+ dictSize = dict->dictSize;
+
+ memmove(safeBuffer, previousDictEnd - dictSize, dictSize);
+
+ dict->dictionary = (const BYTE *)safeBuffer;
+ dict->dictSize = (U32)dictSize;
+
+ return dictSize;
+}
+EXPORT_SYMBOL(LZ4_saveDict);
+
+int LZ4_compress_fast_continue(LZ4_stream_t *LZ4_stream, const char *source,
+ char *dest, int inputSize, int maxOutputSize, int acceleration)
+{
+ LZ4_stream_t_internal *streamPtr = &LZ4_stream->internal_donotuse;
+ const BYTE * const dictEnd = streamPtr->dictionary
+ + streamPtr->dictSize;
+
+ const BYTE *smallest = (const BYTE *) source;
+
+ if (streamPtr->initCheck) {
+ /* Uninitialized structure detected */
+ return 0;
+ }
+
+ if ((streamPtr->dictSize > 0) && (smallest > dictEnd))
+ smallest = dictEnd;
+
+ LZ4_renormDictT(streamPtr, smallest);
- *dst_len = out_len;
+ if (acceleration < 1)
+ acceleration = LZ4_ACCELERATION_DEFAULT;
- return 0;
-exit:
- return ret;
+ /* Check overlapping input/dictionary space */
+ {
+ const BYTE *sourceEnd = (const BYTE *) source + inputSize;
+
+ if ((sourceEnd > streamPtr->dictionary)
+ && (sourceEnd < dictEnd)) {
+ streamPtr->dictSize = (U32)(dictEnd - sourceEnd);
+ if (streamPtr->dictSize > 64 * KB)
+ streamPtr->dictSize = 64 * KB;
+ if (streamPtr->dictSize < 4)
+ streamPtr->dictSize = 0;
+ streamPtr->dictionary = dictEnd - streamPtr->dictSize;
+ }
+ }
+
+ /* prefix mode : source data follows dictionary */
+ if (dictEnd == (const BYTE *)source) {
+ int result;
+
+ if ((streamPtr->dictSize < 64 * KB) &&
+ (streamPtr->dictSize < streamPtr->currentOffset)) {
+ result = LZ4_compress_generic(
+ streamPtr, source, dest, inputSize,
+ maxOutputSize, limitedOutput, byU32,
+ withPrefix64k, dictSmall, acceleration);
+ } else {
+ result = LZ4_compress_generic(
+ streamPtr, source, dest, inputSize,
+ maxOutputSize, limitedOutput, byU32,
+ withPrefix64k, noDictIssue, acceleration);
+ }
+ streamPtr->dictSize += (U32)inputSize;
+ streamPtr->currentOffset += (U32)inputSize;
+ return result;
+ }
+
+ /* external dictionary mode */
+ {
+ int result;
+
+ if ((streamPtr->dictSize < 64 * KB) &&
+ (streamPtr->dictSize < streamPtr->currentOffset)) {
+ result = LZ4_compress_generic(
+ streamPtr, source, dest, inputSize,
+ maxOutputSize, limitedOutput, byU32,
+ usingExtDict, dictSmall, acceleration);
+ } else {
+ result = LZ4_compress_generic(
+ streamPtr, source, dest, inputSize,
+ maxOutputSize, limitedOutput, byU32,
+ usingExtDict, noDictIssue, acceleration);
+ }
+ streamPtr->dictionary = (const BYTE *)source;
+ streamPtr->dictSize = (U32)inputSize;
+ streamPtr->currentOffset += (U32)inputSize;
+ return result;
+ }
+}
+EXPORT_SYMBOL(LZ4_compress_fast_continue);
+
+/*-******************************
+ * For backwards compatibility
+ ********************************/
+int lz4_compress(const unsigned char *src, size_t src_len, unsigned char *dst,
+ size_t *dst_len, void *wrkmem) {
+ *dst_len = LZ4_compress_default(src, dst, src_len,
+ *dst_len, wrkmem);
+
+ /*
+ * Prior lz4_compress will return -1 in case of error
+ * and 0 on success
+ * while new LZ4_compress_fast/default
+ * returns 0 in case of error
+ * and the output length on success
+ */
+ if (!*dst_len)
+ return -1;
+ else
+ return 0;
}
EXPORT_SYMBOL(lz4_compress);
diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c
index 6d940c7..fd665ca 100644
--- a/lib/lz4/lz4_decompress.c
+++ b/lib/lz4/lz4_decompress.c
@@ -1,25 +1,16 @@
/*
- * LZ4 Decompressor for Linux kernel
- *
- * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
- *
- * Based on LZ4 implementation by Yann Collet.
- *
* LZ4 - Fast LZ compression algorithm
- * Copyright (C) 2011-2012, Yann Collet.
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
- *
+ * Copyright (C) 2011 - 2016, Yann Collet.
+ * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php)
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
- *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -31,313 +22,508 @@
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
*
- * You can contact the author at :
- * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- * - LZ4 source repository : http://code.google.com/p/lz4/
+ * Changed for kernel usage by:
+ * Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
*/
-#ifndef STATIC
+/*-************************************
+ * Dependencies
+ **************************************/
+#include <linux/lz4.h>
+#include "lz4defs.h"
+#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
-#endif
-#include <linux/lz4.h>
-
#include <asm/unaligned.h>
-#include "lz4defs.h"
-
-static const int dec32table[] = {0, 3, 2, 3, 0, 0, 0, 0};
-#if LZ4_ARCH64
-static const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
-#endif
-
-static int lz4_uncompress(const char *source, char *dest, int osize)
+/*-*****************************
+ * Decompression functions
+ *******************************/
+/* LZ4_decompress_generic() :
+ * This generic decompression function cover all use cases.
+ * It shall be instantiated several times, using different sets of directives
+ * Note that it is important this generic function is really inlined,
+ * in order to remove useless branches during compilation optimization.
+ */
+static inline int LZ4_decompress_generic(
+ const char *const source,
+ char * const dest,
+ int inputSize,
+ /*
+ * If endOnInput == endOnInputSize,
+ * this value is the max size of Output Buffer.
+ */
+ int outputSize,
+ /* endOnOutputSize, endOnInputSize */
+ int endOnInput,
+ /* full, partial */
+ int partialDecoding,
+ /* only used if partialDecoding == partial */
+ int targetOutputSize,
+ /* noDict, withPrefix64k, usingExtDict */
+ int dict,
+ /* == dest when no prefix */
+ const BYTE * const lowPrefix,
+ /* only if dict == usingExtDict */
+ const BYTE * const dictStart,
+ /* note : = 0 if noDict */
+ const size_t dictSize
+ )
{
+ /* Local Variables */
const BYTE *ip = (const BYTE *) source;
- const BYTE *ref;
+ const BYTE * const iend = ip + inputSize;
+
BYTE *op = (BYTE *) dest;
- BYTE * const oend = op + osize;
+ BYTE * const oend = op + outputSize;
BYTE *cpy;
- unsigned token;
- size_t length;
+ BYTE *oexit = op + targetOutputSize;
+ const BYTE * const lowLimit = lowPrefix - dictSize;
+
+ const BYTE * const dictEnd = (const BYTE *)dictStart + dictSize;
+ const unsigned int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4};
+ const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3};
+
+ const int safeDecode = (endOnInput == endOnInputSize);
+ const int checkOffset = ((safeDecode) && (dictSize < (int)(64 * KB)));
+ /* Special cases */
+ /* targetOutputSize too high => decode everything */
+ if ((partialDecoding) && (oexit > oend - MFLIMIT))
+ oexit = oend - MFLIMIT;
+
+ /* Empty output buffer */
+ if ((endOnInput) && (unlikely(outputSize == 0)))
+ return ((inputSize == 1) && (*ip == 0)) ? 0 : -1;
+
+ if ((!endOnInput) && (unlikely(outputSize == 0)))
+ return (*ip == 0 ? 1 : -1);
+
+ /* Main Loop : decode sequences */
while (1) {
+ size_t length;
+ const BYTE *match;
+ size_t offset;
+
+ /* get literal length */
+ unsigned int const token = *ip++;
+
+ length = token>>ML_BITS;
- /* get runlength */
- token = *ip++;
- length = (token >> ML_BITS);
if (length == RUN_MASK) {
- size_t len;
+ unsigned int s;
- len = *ip++;
- for (; len == 255; length += 255)
- len = *ip++;
- if (unlikely(length > (size_t)(length + len)))
+ do {
+ s = *ip++;
+ length += s;
+ } while (likely(endOnInput
+ ? ip < iend - RUN_MASK
+ : 1) & (s == 255));
+
+ if ((safeDecode)
+ && unlikely(
+ (size_t)(op + length) < (size_t)(op))) {
+ /* overflow detection */
goto _output_error;
- length += len;
+ }
+ if ((safeDecode)
+ && unlikely(
+ (size_t)(ip + length) < (size_t)(ip))) {
+ /* overflow detection */
+ goto _output_error;
+ }
}
/* copy literals */
cpy = op + length;
- if (unlikely(cpy > oend - COPYLENGTH)) {
- /*
- * Error: not enough place for another match
- * (min 4) + 5 literals
- */
- if (cpy != oend)
- goto _output_error;
-
+ if (((endOnInput) && ((cpy > (partialDecoding ? oexit : oend - MFLIMIT))
+ || (ip + length > iend - (2 + 1 + LASTLITERALS))))
+ || ((!endOnInput) && (cpy > oend - WILDCOPYLENGTH))) {
+ if (partialDecoding) {
+ if (cpy > oend) {
+ /*
+ * Error :
+ * write attempt beyond end of output buffer
+ */
+ goto _output_error;
+ }
+ if ((endOnInput)
+ && (ip + length > iend)) {
+ /*
+ * Error :
+ * read attempt beyond
+ * end of input buffer
+ */
+ goto _output_error;
+ }
+ } else {
+ if ((!endOnInput)
+ && (cpy != oend)) {
+ /*
+ * Error :
+ * block decoding must
+ * stop exactly there
+ */
+ goto _output_error;
+ }
+ if ((endOnInput)
+ && ((ip + length != iend)
+ || (cpy > oend))) {
+ /*
+ * Error :
+ * input must be consumed
+ */
+ goto _output_error;
+ }
+ }
memcpy(op, ip, length);
ip += length;
- break; /* EOF */
+ op += length;
+ /* Necessarily EOF, due to parsing restrictions */
+ break;
}
- LZ4_WILDCOPY(ip, op, cpy);
- ip -= (op - cpy);
- op = cpy;
+ LZ4_wildCopy(op, ip, cpy);
+ ip += length; op = cpy;
/* get offset */
- LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip);
- ip += 2;
-
- /* Error: offset create reference outside destination buffer */
- if (unlikely(ref < (BYTE *const) dest))
+ offset = LZ4_readLE16(ip); ip += 2;
+ match = op - offset;
+ if ((checkOffset) && (unlikely(match < lowLimit))) {
+ /* Error : offset outside buffers */
goto _output_error;
+ }
+ /* costs ~1%; silence an msan warning when offset == 0 */
+ LZ4_write32(op, (U32)offset);
/* get matchlength */
length = token & ML_MASK;
if (length == ML_MASK) {
- for (; *ip == 255; length += 255)
- ip++;
- if (unlikely(length > (size_t)(length + *ip)))
+ unsigned int s;
+
+ do {
+ s = *ip++;
+ if ((endOnInput) && (ip > iend - LASTLITERALS))
+ goto _output_error;
+ length += s;
+ } while (s == 255);
+ if ((safeDecode)
+ && unlikely(
+ (size_t)(op + length) < (size_t)op)) {
+ /* overflow detection */
goto _output_error;
- length += *ip++;
+ }
}
+ length += MINMATCH;
+
+ /* check external dictionary */
+ if ((dict == usingExtDict) && (match < lowPrefix)) {
+ if (unlikely(op + length > oend - LASTLITERALS)) {
+ /* doesn't respect parsing restriction */
+ goto _output_error;
+ }
- /* copy repeated sequence */
- if (unlikely((op - ref) < STEPSIZE)) {
-#if LZ4_ARCH64
- int dec64 = dec64table[op - ref];
-#else
- const int dec64 = 0;
-#endif
- op[0] = ref[0];
- op[1] = ref[1];
- op[2] = ref[2];
- op[3] = ref[3];
- op += 4;
- ref += 4;
- ref -= dec32table[op-ref];
- PUT4(ref, op);
- op += STEPSIZE - 4;
- ref -= dec64;
+ if (length <= (size_t)(lowPrefix - match)) {
+ /*
+ * match can be copied as a single segment
+ * from external dictionary
+ */
+ memmove(op, dictEnd - (lowPrefix - match), length);
+ op += length;
+ } else {
+ /*
+ * match encompass external
+ * dictionary and current block
+ */
+ size_t const copySize = (size_t)(lowPrefix - match);
+ size_t const restSize = length - copySize;
+
+ memcpy(op, dictEnd - copySize, copySize);
+ op += copySize;
+
+ if (restSize > (size_t)(op - lowPrefix)) {
+ /* overlap copy */
+ BYTE * const endOfMatch = op + restSize;
+ const BYTE *copyFrom = lowPrefix;
+
+ while (op < endOfMatch)
+ *op++ = *copyFrom++;
+ } else {
+ memcpy(op, lowPrefix, restSize);
+ op += restSize;
+ }
+ }
+ continue;
+ }
+
+ /* copy match within block */
+ cpy = op + length;
+ if (unlikely(offset < 8)) {
+ const int dec64 = dec64table[offset];
+
+ op[0] = match[0];
+ op[1] = match[1];
+ op[2] = match[2];
+ op[3] = match[3];
+ match += dec32table[offset];
+ memcpy(op + 4, match, 4);
+ match -= dec64;
} else {
- LZ4_COPYSTEP(ref, op);
+ LZ4_copy8(op, match); match += 8;
}
- cpy = op + length - (STEPSIZE - 4);
- if (cpy > (oend - COPYLENGTH)) {
- /* Error: request to write beyond destination buffer */
- if (cpy > oend)
- goto _output_error;
-#if LZ4_ARCH64
- if ((ref + COPYLENGTH) > oend)
-#else
- if ((ref + COPYLENGTH) > oend ||
- (op + COPYLENGTH) > oend)
-#endif
+ op += 8;
+
+ if (unlikely(cpy > oend - 12)) {
+ BYTE * const oCopyLimit = oend - (WILDCOPYLENGTH - 1);
+
+ if (cpy > oend - LASTLITERALS) {
+ /*
+ * Error : last LASTLITERALS bytes
+ * must be literals (uncompressed)
+ */
goto _output_error;
- LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH));
+ }
+ if (op < oCopyLimit) {
+ LZ4_wildCopy(op, match, oCopyLimit);
+ match += oCopyLimit - op;
+ op = oCopyLimit;
+ }
while (op < cpy)
- *op++ = *ref++;
- op = cpy;
- /*
- * Check EOF (should never happen, since last 5 bytes
- * are supposed to be literals)
- */
- if (op == oend)
- goto _output_error;
- continue;
+ *op++ = *match++;
+ } else {
+ LZ4_copy8(op, match);
+ if (length > 16)
+ LZ4_wildCopy(op + 8, match + 8, cpy);
}
- LZ4_SECURECOPY(ref, op, cpy);
op = cpy; /* correction */
}
+
/* end of decoding */
- return (int) (((char *)ip) - source);
+ if (endOnInput) {
+ /* Nb of output bytes decoded */
+ return (int) (((char *)op) - dest);
+ } else {
+ /* Nb of input bytes read */
+ return (int) (((const char *)ip) - source);
+ }
- /* write overflow error detected */
+ /* Overflow error detected */
_output_error:
return -1;
}
-static int lz4_uncompress_unknownoutputsize(const char *source, char *dest,
- int isize, size_t maxoutputsize)
+int LZ4_decompress_safe(const char *source, char *dest,
+ int compressedSize, int maxDecompressedSize)
{
- const BYTE *ip = (const BYTE *) source;
- const BYTE *const iend = ip + isize;
- const BYTE *ref;
-
+ return LZ4_decompress_generic(source, dest, compressedSize,
+ maxDecompressedSize, endOnInputSize, full, 0,
+ noDict, (BYTE *)dest, NULL, 0);
+}
+EXPORT_SYMBOL(LZ4_decompress_safe);
- BYTE *op = (BYTE *) dest;
- BYTE * const oend = op + maxoutputsize;
- BYTE *cpy;
+int LZ4_decompress_safe_partial(const char *source, char *dest,
+ int compressedSize, int targetOutputSize, int maxDecompressedSize)
+{
+ return LZ4_decompress_generic(source, dest, compressedSize,
+ maxDecompressedSize, endOnInputSize, partial,
+ targetOutputSize, noDict, (BYTE *)dest, NULL, 0);
+}
+EXPORT_SYMBOL(LZ4_decompress_safe_partial);
- /* Main Loop */
- while (ip < iend) {
+int LZ4_decompress_fast(const char *source, char *dest, int originalSize)
+{
+ return LZ4_decompress_generic(source, dest, 0, originalSize,
+ endOnOutputSize, full, 0, withPrefix64k,
+ (BYTE *)(dest - 64 * KB), NULL, 64 * KB);
+}
+EXPORT_SYMBOL(LZ4_decompress_fast);
- unsigned token;
- size_t length;
+int LZ4_setStreamDecode(LZ4_streamDecode_t *LZ4_streamDecode,
+ const char *dictionary, int dictSize)
+{
+ LZ4_streamDecode_t_internal *lz4sd = (LZ4_streamDecode_t_internal *) LZ4_streamDecode;
- /* get runlength */
- token = *ip++;
- length = (token >> ML_BITS);
- if (length == RUN_MASK) {
- int s = 255;
- while ((ip < iend) && (s == 255)) {
- s = *ip++;
- if (unlikely(length > (size_t)(length + s)))
- goto _output_error;
- length += s;
- }
- }
- /* copy literals */
- cpy = op + length;
- if ((cpy > oend - COPYLENGTH) ||
- (ip + length > iend - COPYLENGTH)) {
-
- if (cpy > oend)
- goto _output_error;/* writes beyond buffer */
-
- if (ip + length != iend)
- goto _output_error;/*
- * Error: LZ4 format requires
- * to consume all input
- * at this stage
- */
- memcpy(op, ip, length);
- op += length;
- break;/* Necessarily EOF, due to parsing restrictions */
- }
- LZ4_WILDCOPY(ip, op, cpy);
- ip -= (op - cpy);
- op = cpy;
+ lz4sd->prefixSize = (size_t) dictSize;
+ lz4sd->prefixEnd = (const BYTE *) dictionary + dictSize;
+ lz4sd->externalDict = NULL;
+ lz4sd->extDictSize = 0;
+ return 1;
+}
+EXPORT_SYMBOL(LZ4_setStreamDecode);
- /* get offset */
- LZ4_READ_LITTLEENDIAN_16(ref, cpy, ip);
- ip += 2;
- if (ref < (BYTE * const) dest)
- goto _output_error;
- /*
- * Error : offset creates reference
- * outside of destination buffer
- */
+/*
+ * *_continue() :
+ * These decoding functions allow decompression of multiple blocks
+ * in "streaming" mode.
+ * Previously decoded blocks must still be available at the memory
+ * position where they were decoded.
+ * If it's not possible, save the relevant part of
+ * decoded data into a safe buffer,
+ * and indicate where it stands using LZ4_setStreamDecode()
+ */
+int LZ4_decompress_safe_continue(LZ4_streamDecode_t *LZ4_streamDecode,
+ const char *source, char *dest, int compressedSize, int maxOutputSize)
+{
+ LZ4_streamDecode_t_internal *lz4sd = &LZ4_streamDecode->internal_donotuse;
+ int result;
+
+ if (lz4sd->prefixEnd == (BYTE *)dest) {
+ result = LZ4_decompress_generic(source, dest,
+ compressedSize,
+ maxOutputSize,
+ endOnInputSize, full, 0,
+ usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize,
+ lz4sd->externalDict,
+ lz4sd->extDictSize);
+
+ if (result <= 0)
+ return result;
+
+ lz4sd->prefixSize += result;
+ lz4sd->prefixEnd += result;
+ } else {
+ lz4sd->extDictSize = lz4sd->prefixSize;
+ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
+ result = LZ4_decompress_generic(source, dest,
+ compressedSize, maxOutputSize,
+ endOnInputSize, full, 0,
+ usingExtDict, (BYTE *)dest,
+ lz4sd->externalDict, lz4sd->extDictSize);
+ if (result <= 0)
+ return result;
+ lz4sd->prefixSize = result;
+ lz4sd->prefixEnd = (BYTE *)dest + result;
+ }
- /* get matchlength */
- length = (token & ML_MASK);
- if (length == ML_MASK) {
- while (ip < iend) {
- int s = *ip++;
- if (unlikely(length > (size_t)(length + s)))
- goto _output_error;
- length += s;
- if (s == 255)
- continue;
- break;
- }
- }
+ return result;
+}
+EXPORT_SYMBOL(LZ4_decompress_safe_continue);
- /* copy repeated sequence */
- if (unlikely((op - ref) < STEPSIZE)) {
-#if LZ4_ARCH64
- int dec64 = dec64table[op - ref];
-#else
- const int dec64 = 0;
-#endif
- op[0] = ref[0];
- op[1] = ref[1];
- op[2] = ref[2];
- op[3] = ref[3];
- op += 4;
- ref += 4;
- ref -= dec32table[op - ref];
- PUT4(ref, op);
- op += STEPSIZE - 4;
- ref -= dec64;
- } else {
- LZ4_COPYSTEP(ref, op);
- }
- cpy = op + length - (STEPSIZE-4);
- if (cpy > oend - COPYLENGTH) {
- if (cpy > oend)
- goto _output_error; /* write outside of buf */
-#if LZ4_ARCH64
- if ((ref + COPYLENGTH) > oend)
-#else
- if ((ref + COPYLENGTH) > oend ||
- (op + COPYLENGTH) > oend)
-#endif
- goto _output_error;
- LZ4_SECURECOPY(ref, op, (oend - COPYLENGTH));
- while (op < cpy)
- *op++ = *ref++;
- op = cpy;
- /*
- * Check EOF (should never happen, since last 5 bytes
- * are supposed to be literals)
- */
- if (op == oend)
- goto _output_error;
- continue;
- }
- LZ4_SECURECOPY(ref, op, cpy);
- op = cpy; /* correction */
+int LZ4_decompress_fast_continue(LZ4_streamDecode_t *LZ4_streamDecode,
+ const char *source, char *dest, int originalSize)
+{
+ LZ4_streamDecode_t_internal *lz4sd = &LZ4_streamDecode->internal_donotuse;
+ int result;
+
+ if (lz4sd->prefixEnd == (BYTE *)dest) {
+ result = LZ4_decompress_generic(source, dest, 0, originalSize,
+ endOnOutputSize, full, 0,
+ usingExtDict,
+ lz4sd->prefixEnd - lz4sd->prefixSize,
+ lz4sd->externalDict, lz4sd->extDictSize);
+
+ if (result <= 0)
+ return result;
+
+ lz4sd->prefixSize += originalSize;
+ lz4sd->prefixEnd += originalSize;
+ } else {
+ lz4sd->extDictSize = lz4sd->prefixSize;
+ lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize;
+ result = LZ4_decompress_generic(source, dest, 0, originalSize,
+ endOnOutputSize, full, 0,
+ usingExtDict, (BYTE *)dest,
+ lz4sd->externalDict, lz4sd->extDictSize);
+ if (result <= 0)
+ return result;
+ lz4sd->prefixSize = originalSize;
+ lz4sd->prefixEnd = (BYTE *)dest + originalSize;
}
- /* end of decoding */
- return (int) (((char *) op) - dest);
- /* write overflow error detected */
-_output_error:
- return -1;
+ return result;
}
+EXPORT_SYMBOL(LZ4_decompress_fast_continue);
-int lz4_decompress(const unsigned char *src, size_t *src_len,
- unsigned char *dest, size_t actual_dest_len)
+/*
+ * Advanced decoding functions :
+ * *_usingDict() :
+ * These decoding functions work the same as "_continue" ones,
+ * the dictionary must be explicitly provided within parameters
+ */
+static inline int LZ4_decompress_usingDict_generic(const char *source,
+ char *dest, int compressedSize, int maxOutputSize, int safe,
+ const char *dictStart, int dictSize)
{
- int ret = -1;
- int input_len = 0;
-
- input_len = lz4_uncompress(src, dest, actual_dest_len);
- if (input_len < 0)
- goto exit_0;
- *src_len = input_len;
+ if (dictSize == 0)
+ return LZ4_decompress_generic(source, dest,
+ compressedSize, maxOutputSize, safe, full, 0,
+ noDict, (BYTE *)dest, NULL, 0);
+ if (dictStart + dictSize == dest) {
+ if (dictSize >= (int)(64 * KB - 1))
+ return LZ4_decompress_generic(source, dest,
+ compressedSize, maxOutputSize, safe, full, 0,
+ withPrefix64k, (BYTE *)dest - 64 * KB, NULL, 0);
+ return LZ4_decompress_generic(source, dest, compressedSize,
+ maxOutputSize, safe, full, 0, noDict,
+ (BYTE *)dest - dictSize, NULL, 0);
+ }
+ return LZ4_decompress_generic(source, dest, compressedSize,
+ maxOutputSize, safe, full, 0, usingExtDict,
+ (BYTE *)dest, (const BYTE *)dictStart, dictSize);
+}
- return 0;
-exit_0:
- return ret;
+int LZ4_decompress_safe_usingDict(const char *source, char *dest,
+ int compressedSize, int maxOutputSize,
+ const char *dictStart, int dictSize)
+{
+ return LZ4_decompress_usingDict_generic(source, dest,
+ compressedSize, maxOutputSize, 1, dictStart, dictSize);
}
-#ifndef STATIC
-EXPORT_SYMBOL(lz4_decompress);
-#endif
+EXPORT_SYMBOL(LZ4_decompress_safe_usingDict);
-int lz4_decompress_unknownoutputsize(const unsigned char *src, size_t src_len,
- unsigned char *dest, size_t *dest_len)
+int LZ4_decompress_fast_usingDict(const char *source, char *dest,
+ int originalSize, const char *dictStart, int dictSize)
{
- int ret = -1;
- int out_len = 0;
-
- out_len = lz4_uncompress_unknownoutputsize(src, dest, src_len,
- *dest_len);
- if (out_len < 0)
- goto exit_0;
- *dest_len = out_len;
-
- return 0;
-exit_0:
- return ret;
+ return LZ4_decompress_usingDict_generic(source, dest, 0,
+ originalSize, 0, dictStart, dictSize);
+}
+EXPORT_SYMBOL(LZ4_decompress_fast_usingDict);
+
+/*-******************************
+ * For backwards compatibility
+ ********************************/
+int lz4_decompress_unknownoutputsize(const unsigned char *src,
+ size_t src_len, unsigned char *dest, size_t *dest_len) {
+ *dest_len = LZ4_decompress_safe(src, dest,
+ src_len, *dest_len);
+
+ /*
+ * Prior lz4_decompress_unknownoutputsize will return
+ * 0 for success and a negative result for error
+ * new LZ4_decompress_safe returns
+ * - the length of data read on success
+ * - and also a negative result on error
+ * meaning when result > 0, we just return 0 here
+ */
+ if (src_len > 0)
+ return 0;
+ else
+ return -1;
}
-#ifndef STATIC
EXPORT_SYMBOL(lz4_decompress_unknownoutputsize);
+int lz4_decompress(const unsigned char *src, size_t *src_len,
+ unsigned char *dest, size_t actual_dest_len) {
+ *src_len = LZ4_decompress_fast(src, dest, actual_dest_len);
+
+ /*
+ * Prior lz4_decompress will return
+ * 0 for success and a negative result for error
+ * new LZ4_decompress_fast returns
+ * - the length of data read on success
+ * - and also a negative result on error
+ * meaning when result > 0, we just return 0 here
+ */
+ if (*src_len > 0)
+ return 0;
+ else
+ return -1;
+}
+EXPORT_SYMBOL(lz4_decompress);
+
MODULE_LICENSE("Dual BSD/GPL");
-MODULE_DESCRIPTION("LZ4 Decompressor");
-#endif
+MODULE_DESCRIPTION("LZ4 decompressor");
diff --git a/lib/lz4/lz4defs.h b/lib/lz4/lz4defs.h
index c79d7ea..23e1a1b 100644
--- a/lib/lz4/lz4defs.h
+++ b/lib/lz4/lz4defs.h
@@ -1,157 +1,229 @@
+#ifndef __LZ4DEFS_H__
+#define __LZ4DEFS_H__
+
/*
- * lz4defs.h -- architecture specific defines
- *
- * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
+ * lz4defs.h -- common and architecture specific defines for the kernel usage
+
+ * LZ4 - Fast LZ compression algorithm
+ * Copyright (C) 2011-2016, Yann Collet.
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * You can contact the author at :
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Changed for kernel usage by:
+ * Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
*/
+#include <asm/unaligned.h>
+#include <linux/string.h> /* memset, memcpy */
+
/*
* Detects 64 bits mode
- */
+*/
#if defined(CONFIG_64BIT)
#define LZ4_ARCH64 1
#else
#define LZ4_ARCH64 0
#endif
-/*
- * Architecture-specific macros
- */
-#define BYTE u8
-typedef struct _U16_S { u16 v; } U16_S;
-typedef struct _U32_S { u32 v; } U32_S;
-typedef struct _U64_S { u64 v; } U64_S;
-#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
-
-#define A16(x) (((U16_S *)(x))->v)
-#define A32(x) (((U32_S *)(x))->v)
-#define A64(x) (((U64_S *)(x))->v)
-
-#define PUT4(s, d) (A32(d) = A32(s))
-#define PUT8(s, d) (A64(d) = A64(s))
-
-#define LZ4_READ_LITTLEENDIAN_16(d, s, p) \
- (d = s - A16(p))
-
-#define LZ4_WRITE_LITTLEENDIAN_16(p, v) \
- do { \
- A16(p) = v; \
- p += 2; \
- } while (0)
-#else /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
-
-#define A64(x) get_unaligned((u64 *)&(((U16_S *)(x))->v))
-#define A32(x) get_unaligned((u32 *)&(((U16_S *)(x))->v))
-#define A16(x) get_unaligned((u16 *)&(((U16_S *)(x))->v))
-
-#define PUT4(s, d) \
- put_unaligned(get_unaligned((const u32 *) s), (u32 *) d)
-#define PUT8(s, d) \
- put_unaligned(get_unaligned((const u64 *) s), (u64 *) d)
-
-#define LZ4_READ_LITTLEENDIAN_16(d, s, p) \
- (d = s - get_unaligned_le16(p))
-
-#define LZ4_WRITE_LITTLEENDIAN_16(p, v) \
- do { \
- put_unaligned_le16(v, (u16 *)(p)); \
- p += 2; \
- } while (0)
-#endif
+/*-************************************
+ * Basic Types
+ **************************************/
+#include <linux/types.h>
-#define COPYLENGTH 8
-#define ML_BITS 4
-#define ML_MASK ((1U << ML_BITS) - 1)
-#define RUN_BITS (8 - ML_BITS)
-#define RUN_MASK ((1U << RUN_BITS) - 1)
-#define MEMORY_USAGE 14
-#define MINMATCH 4
-#define SKIPSTRENGTH 6
-#define LASTLITERALS 5
-#define MFLIMIT (COPYLENGTH + MINMATCH)
-#define MINLENGTH (MFLIMIT + 1)
-#define MAXD_LOG 16
-#define MAXD (1 << MAXD_LOG)
-#define MAXD_MASK (u32)(MAXD - 1)
-#define MAX_DISTANCE (MAXD - 1)
-#define HASH_LOG (MAXD_LOG - 1)
-#define HASHTABLESIZE (1 << HASH_LOG)
-#define MAX_NB_ATTEMPTS 256
-#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT - 1))
-#define HASHLOG64K ((MEMORY_USAGE - 2) + 1)
-#define HASH64KTABLESIZE (1U << HASHLOG64K)
-#define LZ4_HASH_VALUE(p) (((A32(p)) * 2654435761U) >> \
- ((MINMATCH * 8) - (MEMORY_USAGE-2)))
-#define LZ4_HASH64K_VALUE(p) (((A32(p)) * 2654435761U) >> \
- ((MINMATCH * 8) - HASHLOG64K))
-#define HASH_VALUE(p) (((A32(p)) * 2654435761U) >> \
- ((MINMATCH * 8) - HASH_LOG))
-
-#if LZ4_ARCH64/* 64-bit */
-#define STEPSIZE 8
-
-#define LZ4_COPYSTEP(s, d) \
- do { \
- PUT8(s, d); \
- d += 8; \
- s += 8; \
- } while (0)
-
-#define LZ4_COPYPACKET(s, d) LZ4_COPYSTEP(s, d)
-
-#define LZ4_SECURECOPY(s, d, e) \
- do { \
- if (d < e) { \
- LZ4_WILDCOPY(s, d, e); \
- } \
- } while (0)
-#define HTYPE u32
-
-#ifdef __BIG_ENDIAN
-#define LZ4_NBCOMMONBYTES(val) (__builtin_clzll(val) >> 3)
+typedef uint8_t BYTE;
+typedef uint16_t U16;
+typedef uint32_t U32;
+typedef int32_t S32;
+typedef uint64_t U64;
+typedef uintptr_t uptrval;
+
+/*-************************************
+ * Constants
+ **************************************/
+#define MINMATCH 4
+
+#define WILDCOPYLENGTH 8
+#define LASTLITERALS 5
+#define MFLIMIT (WILDCOPYLENGTH+MINMATCH)
+static const int LZ4_minLength = (MFLIMIT+1);
+
+#define KB (1<<10)
+#define MB (1<<20)
+#define GB (1U<<30)
+
+#define MAXD_LOG 16
+#define MAX_DISTANCE ((1<<MAXD_LOG) - 1)
+#define STEPSIZE sizeof(size_t)
+
+#define ML_BITS 4
+#define ML_MASK ((1U<<ML_BITS)-1)
+#define RUN_BITS (8-ML_BITS)
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+
+static const int LZ4_64Klimit = ((64 * KB) + (MFLIMIT-1));
+static const U32 LZ4_skipTrigger = 6;
+
+/*-************************************
+ * Reading and writing into memory
+ **************************************/
+
+static inline U16 LZ4_read16(const void *memPtr)
+{
+ U16 val;
+
+ memcpy(&val, memPtr, sizeof(val));
+
+ return val;
+}
+
+static inline U32 LZ4_read32(const void *memPtr)
+{
+ U32 val;
+
+ memcpy(&val, memPtr, sizeof(val));
+
+ return val;
+}
+
+static inline size_t LZ4_read_ARCH(const void *memPtr)
+{
+ size_t val;
+
+ memcpy(&val, memPtr, sizeof(val));
+
+ return val;
+}
+
+static inline void LZ4_write16(void *memPtr, U16 value)
+{
+ memcpy(memPtr, &value, sizeof(value));
+}
+
+static inline void LZ4_write32(void *memPtr, U32 value)
+{
+ memcpy(memPtr, &value, sizeof(value));
+}
+
+static inline U16 LZ4_readLE16(const void *memPtr)
+{
+#ifdef __LITTLE_ENDIAN__
+ return LZ4_read16(memPtr);
#else
-#define LZ4_NBCOMMONBYTES(val) (__builtin_ctzll(val) >> 3)
-#endif
+ const BYTE *p = (const BYTE *)memPtr;
-#else /* 32-bit */
-#define STEPSIZE 4
+ return (U16)((U16)p[0] + (p[1] << 8));
+#endif
+}
-#define LZ4_COPYSTEP(s, d) \
- do { \
- PUT4(s, d); \
- d += 4; \
- s += 4; \
- } while (0)
+static inline void LZ4_writeLE16(void *memPtr, U16 value)
+{
+#ifdef __LITTLE_ENDIAN__
+ LZ4_write16(memPtr, value);
+#else
+ BYTE *p = (BYTE *)memPtr;
-#define LZ4_COPYPACKET(s, d) \
- do { \
- LZ4_COPYSTEP(s, d); \
- LZ4_COPYSTEP(s, d); \
- } while (0)
+ p[0] = (BYTE) value;
+ p[1] = (BYTE)(value>>8);
+#endif
+}
-#define LZ4_SECURECOPY LZ4_WILDCOPY
-#define HTYPE const u8*
+static inline void LZ4_copy8(void *dst, const void *src)
+{
+ memcpy(dst, src, 8);
+}
-#ifdef __BIG_ENDIAN
+/*
+ * customized variant of memcpy,
+ * which can overwrite up to 7 bytes beyond dstEnd
+ */
+static inline void LZ4_wildCopy(void *dstPtr, const void *srcPtr, void *dstEnd)
+{
+ BYTE *d = (BYTE *)dstPtr;
+ const BYTE *s = (const BYTE *)srcPtr;
+ BYTE *const e = (BYTE *)dstEnd;
+
+ do {
+ LZ4_copy8(d, s);
+ d += 8;
+ s += 8;
+ } while (d < e);
+}
+
+#if LZ4_ARCH64
+#ifdef __BIG_ENDIAN__
+#define LZ4_NBCOMMONBYTES(val) (__builtin_clzll(val) >> 3)
+#else
+#define LZ4_NBCOMMONBYTES(val) (__builtin_ctzll(val) >> 3)
+#endif
+#else
+#ifdef __BIG_ENDIAN__
#define LZ4_NBCOMMONBYTES(val) (__builtin_clz(val) >> 3)
#else
#define LZ4_NBCOMMONBYTES(val) (__builtin_ctz(val) >> 3)
#endif
+#endif
+static inline unsigned int LZ4_count(const BYTE *pIn, const BYTE *pMatch,
+ const BYTE *pInLimit)
+{
+ const BYTE *const pStart = pIn;
+
+ while (likely(pIn < pInLimit-(STEPSIZE-1))) {
+ size_t diff = LZ4_read_ARCH(pMatch) ^ LZ4_read_ARCH(pIn);
+
+ if (!diff) {
+ pIn += STEPSIZE;
+ pMatch += STEPSIZE;
+ continue;
+ }
+ pIn += LZ4_NBCOMMONBYTES(diff);
+ return (unsigned int)(pIn - pStart);
+ }
+
+#ifdef LZ4_ARCH64
+ if ((pIn < (pInLimit-3))
+ && (LZ4_read32(pMatch) == LZ4_read32(pIn))) {
+ pIn += 4; pMatch += 4;
+ }
#endif
+ if ((pIn < (pInLimit-1))
+ && (LZ4_read16(pMatch) == LZ4_read16(pIn))) {
+ pIn += 2; pMatch += 2;
+ }
+ if ((pIn < pInLimit) && (*pMatch == *pIn))
+ pIn++;
+ return (unsigned int)(pIn - pStart);
+}
+
+typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
+typedef enum { byPtr, byU32, byU16 } tableType_t;
+
+typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
+typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive;
-#define LZ4_WILDCOPY(s, d, e) \
- do { \
- LZ4_COPYPACKET(s, d); \
- } while (d < e)
-
-#define LZ4_BLINDCOPY(s, d, l) \
- do { \
- u8 *e = (d) + l; \
- LZ4_WILDCOPY(s, d, e); \
- d = e; \
- } while (0)
+typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive;
+typedef enum { full = 0, partial = 1 } earlyEnd_directive;
+
+#endif
diff --git a/lib/lz4/lz4hc_compress.c b/lib/lz4/lz4hc_compress.c
index f344f76..f1d4662 100644
--- a/lib/lz4/lz4hc_compress.c
+++ b/lib/lz4/lz4hc_compress.c
@@ -1,19 +1,17 @@
/*
* LZ4 HC - High Compression Mode of LZ4
- * Copyright (C) 2011-2012, Yann Collet.
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ * Copyright (C) 2011-2015, Yann Collet.
*
+ * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php)
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
- *
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -25,323 +23,361 @@
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
* You can contact the author at :
- * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- * - LZ4 source repository : http://code.google.com/p/lz4/
+ * - LZ4 homepage : http://www.lz4.org
+ * - LZ4 source repository : https://github.com/lz4/lz4
*
- * Changed for kernel use by:
- * Chanho Min <chanho.min@lge.com>
+ * Changed for kernel usage by:
+ * Sven Schmidt <4sschmid@informatik.uni-hamburg.de>
*/
-#include <linux/module.h>
-#include <linux/kernel.h>
+/*-************************************
+ * Dependencies
+ **************************************/
#include <linux/lz4.h>
-#include <asm/unaligned.h>
#include "lz4defs.h"
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/string.h> /* memset */
+
+/* *************************************
+ * Local Constants and types
+ ***************************************/
-struct lz4hc_data {
- const u8 *base;
- HTYPE hashtable[HASHTABLESIZE];
- u16 chaintable[MAXD];
- const u8 *nexttoupdate;
-} __attribute__((__packed__));
+#define OPTIMAL_ML (int)((ML_MASK - 1) + MINMATCH)
-static inline int lz4hc_init(struct lz4hc_data *hc4, const u8 *base)
+#define HASH_FUNCTION(i) (((i) * 2654435761U) \
+ >> ((MINMATCH*8) - LZ4HC_HASH_LOG))
+#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
+
+static U32 LZ4HC_hashPtr(const void *ptr)
+{
+ return HASH_FUNCTION(LZ4_read32(ptr));
+}
+
+/**************************************
+ * HC Compression
+ **************************************/
+static void LZ4HC_init(LZ4HC_CCtx_internal *hc4, const BYTE *start)
{
- memset((void *)hc4->hashtable, 0, sizeof(hc4->hashtable));
- memset(hc4->chaintable, 0xFF, sizeof(hc4->chaintable));
-
-#if LZ4_ARCH64
- hc4->nexttoupdate = base + 1;
-#else
- hc4->nexttoupdate = base;
-#endif
- hc4->base = base;
- return 1;
+ memset((void *)hc4->hashTable, 0, sizeof(hc4->hashTable));
+ memset(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
+ hc4->nextToUpdate = 64 * KB;
+ hc4->base = start - 64 * KB;
+ hc4->end = start;
+ hc4->dictBase = start - 64 * KB;
+ hc4->dictLimit = 64 * KB;
+ hc4->lowLimit = 64 * KB;
}
/* Update chains up to ip (excluded) */
-static inline void lz4hc_insert(struct lz4hc_data *hc4, const u8 *ip)
+static inline void LZ4HC_Insert(LZ4HC_CCtx_internal *hc4,
+ const BYTE *ip)
{
- u16 *chaintable = hc4->chaintable;
- HTYPE *hashtable = hc4->hashtable;
-#if LZ4_ARCH64
+ U16 * const chainTable = hc4->chainTable;
+ U32 * const hashTable = hc4->hashTable;
const BYTE * const base = hc4->base;
-#else
- const int base = 0;
-#endif
+ U32 const target = (U32)(ip - base);
+ U32 idx = hc4->nextToUpdate;
+
+ while (idx < target) {
+ U32 const h = LZ4HC_hashPtr(base + idx);
+ size_t delta = idx - hashTable[h];
- while (hc4->nexttoupdate < ip) {
- const u8 *p = hc4->nexttoupdate;
- size_t delta = p - (hashtable[HASH_VALUE(p)] + base);
if (delta > MAX_DISTANCE)
delta = MAX_DISTANCE;
- chaintable[(size_t)(p) & MAXD_MASK] = (u16)delta;
- hashtable[HASH_VALUE(p)] = (p) - base;
- hc4->nexttoupdate++;
- }
-}
-static inline size_t lz4hc_commonlength(const u8 *p1, const u8 *p2,
- const u8 *const matchlimit)
-{
- const u8 *p1t = p1;
-
- while (p1t < matchlimit - (STEPSIZE - 1)) {
-#if LZ4_ARCH64
- u64 diff = A64(p2) ^ A64(p1t);
-#else
- u32 diff = A32(p2) ^ A32(p1t);
-#endif
- if (!diff) {
- p1t += STEPSIZE;
- p2 += STEPSIZE;
- continue;
- }
- p1t += LZ4_NBCOMMONBYTES(diff);
- return p1t - p1;
- }
-#if LZ4_ARCH64
- if ((p1t < (matchlimit-3)) && (A32(p2) == A32(p1t))) {
- p1t += 4;
- p2 += 4;
- }
-#endif
+ DELTANEXTU16(idx) = (U16)delta;
- if ((p1t < (matchlimit - 1)) && (A16(p2) == A16(p1t))) {
- p1t += 2;
- p2 += 2;
+ hashTable[h] = idx;
+ idx++;
}
- if ((p1t < matchlimit) && (*p2 == *p1t))
- p1t++;
- return p1t - p1;
+
+ hc4->nextToUpdate = target;
}
-static inline int lz4hc_insertandfindbestmatch(struct lz4hc_data *hc4,
- const u8 *ip, const u8 *const matchlimit, const u8 **matchpos)
+static inline int LZ4HC_InsertAndFindBestMatch(
+ LZ4HC_CCtx_internal *hc4, /* Index table will be updated */
+ const BYTE *ip,
+ const BYTE * const iLimit,
+ const BYTE **matchpos,
+ const int maxNbAttempts)
{
- u16 *const chaintable = hc4->chaintable;
- HTYPE *const hashtable = hc4->hashtable;
- const u8 *ref;
-#if LZ4_ARCH64
+ U16 * const chainTable = hc4->chainTable;
+ U32 * const HashTable = hc4->hashTable;
const BYTE * const base = hc4->base;
-#else
- const int base = 0;
-#endif
- int nbattempts = MAX_NB_ATTEMPTS;
- size_t repl = 0, ml = 0;
- u16 delta;
+ const BYTE * const dictBase = hc4->dictBase;
+ const U32 dictLimit = hc4->dictLimit;
+ const U32 lowLimit = (hc4->lowLimit + 64 * KB > (U32)(ip - base))
+ ? hc4->lowLimit
+ : (U32)(ip - base) - (64 * KB - 1);
+ U32 matchIndex;
+ int nbAttempts = maxNbAttempts;
+ size_t ml = 0;
/* HC4 match finder */
- lz4hc_insert(hc4, ip);
- ref = hashtable[HASH_VALUE(ip)] + base;
-
- /* potential repetition */
- if (ref >= ip-4) {
- /* confirmed */
- if (A32(ref) == A32(ip)) {
- delta = (u16)(ip-ref);
- repl = ml = lz4hc_commonlength(ip + MINMATCH,
- ref + MINMATCH, matchlimit) + MINMATCH;
- *matchpos = ref;
- }
- ref -= (size_t)chaintable[(size_t)(ref) & MAXD_MASK];
- }
+ LZ4HC_Insert(hc4, ip);
+ matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+
+ while ((matchIndex >= lowLimit)
+ && (nbAttempts)) {
+ nbAttempts--;
+ if (matchIndex >= dictLimit) {
+ const BYTE * const match = base + matchIndex;
+
+ if (*(match + ml) == *(ip + ml)
+ && (LZ4_read32(match) == LZ4_read32(ip))) {
+ size_t const mlt = LZ4_count(ip + MINMATCH,
+ match + MINMATCH, iLimit) + MINMATCH;
- while ((ref >= ip - MAX_DISTANCE) && nbattempts) {
- nbattempts--;
- if (*(ref + ml) == *(ip + ml)) {
- if (A32(ref) == A32(ip)) {
- size_t mlt =
- lz4hc_commonlength(ip + MINMATCH,
- ref + MINMATCH, matchlimit) + MINMATCH;
if (mlt > ml) {
ml = mlt;
- *matchpos = ref;
+ *matchpos = match;
+ }
+ }
+ } else {
+ const BYTE * const match = dictBase + matchIndex;
+
+ if (LZ4_read32(match) == LZ4_read32(ip)) {
+ size_t mlt;
+ const BYTE *vLimit = ip
+ + (dictLimit - matchIndex);
+
+ if (vLimit > iLimit)
+ vLimit = iLimit;
+ mlt = LZ4_count(ip + MINMATCH,
+ match + MINMATCH, vLimit) + MINMATCH;
+ if ((ip + mlt == vLimit)
+ && (vLimit < iLimit))
+ mlt += LZ4_count(ip + mlt,
+ base + dictLimit,
+ iLimit);
+ if (mlt > ml) {
+ /* virtual matchpos */
+ ml = mlt;
+ *matchpos = base + matchIndex;
}
}
}
- ref -= (size_t)chaintable[(size_t)(ref) & MAXD_MASK];
- }
-
- /* Complete table */
- if (repl) {
- const BYTE *ptr = ip;
- const BYTE *end;
- end = ip + repl - (MINMATCH-1);
- /* Pre-Load */
- while (ptr < end - delta) {
- chaintable[(size_t)(ptr) & MAXD_MASK] = delta;
- ptr++;
- }
- do {
- chaintable[(size_t)(ptr) & MAXD_MASK] = delta;
- /* Head of chain */
- hashtable[HASH_VALUE(ptr)] = (ptr) - base;
- ptr++;
- } while (ptr < end);
- hc4->nexttoupdate = end;
+ matchIndex -= DELTANEXTU16(matchIndex);
}
return (int)ml;
}
-static inline int lz4hc_insertandgetwidermatch(struct lz4hc_data *hc4,
- const u8 *ip, const u8 *startlimit, const u8 *matchlimit, int longest,
- const u8 **matchpos, const u8 **startpos)
+static inline int LZ4HC_InsertAndGetWiderMatch(
+ LZ4HC_CCtx_internal *hc4,
+ const BYTE * const ip,
+ const BYTE * const iLowLimit,
+ const BYTE * const iHighLimit,
+ int longest,
+ const BYTE **matchpos,
+ const BYTE **startpos,
+ const int maxNbAttempts)
{
- u16 *const chaintable = hc4->chaintable;
- HTYPE *const hashtable = hc4->hashtable;
-#if LZ4_ARCH64
+ U16 * const chainTable = hc4->chainTable;
+ U32 * const HashTable = hc4->hashTable;
const BYTE * const base = hc4->base;
-#else
- const int base = 0;
-#endif
- const u8 *ref;
- int nbattempts = MAX_NB_ATTEMPTS;
- int delta = (int)(ip - startlimit);
+ const U32 dictLimit = hc4->dictLimit;
+ const BYTE * const lowPrefixPtr = base + dictLimit;
+ const U32 lowLimit = (hc4->lowLimit + 64 * KB > (U32)(ip - base))
+ ? hc4->lowLimit
+ : (U32)(ip - base) - (64 * KB - 1);
+ const BYTE * const dictBase = hc4->dictBase;
+ U32 matchIndex;
+ int nbAttempts = maxNbAttempts;
+ int delta = (int)(ip - iLowLimit);
/* First Match */
- lz4hc_insert(hc4, ip);
- ref = hashtable[HASH_VALUE(ip)] + base;
-
- while ((ref >= ip - MAX_DISTANCE) && (ref >= hc4->base)
- && (nbattempts)) {
- nbattempts--;
- if (*(startlimit + longest) == *(ref - delta + longest)) {
- if (A32(ref) == A32(ip)) {
- const u8 *reft = ref + MINMATCH;
- const u8 *ipt = ip + MINMATCH;
- const u8 *startt = ip;
-
- while (ipt < matchlimit-(STEPSIZE - 1)) {
- #if LZ4_ARCH64
- u64 diff = A64(reft) ^ A64(ipt);
- #else
- u32 diff = A32(reft) ^ A32(ipt);
- #endif
-
- if (!diff) {
- ipt += STEPSIZE;
- reft += STEPSIZE;
- continue;
+ LZ4HC_Insert(hc4, ip);
+ matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+
+ while ((matchIndex >= lowLimit)
+ && (nbAttempts)) {
+ nbAttempts--;
+ if (matchIndex >= dictLimit) {
+ const BYTE *matchPtr = base + matchIndex;
+
+ if (*(iLowLimit + longest)
+ == *(matchPtr - delta + longest)) {
+ if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
+ int mlt = MINMATCH + LZ4_count(
+ ip + MINMATCH,
+ matchPtr + MINMATCH,
+ iHighLimit);
+ int back = 0;
+
+ while ((ip + back > iLowLimit)
+ && (matchPtr + back > lowPrefixPtr)
+ && (ip[back - 1] == matchPtr[back - 1]))
+ back--;
+
+ mlt -= back;
+
+ if (mlt > longest) {
+ longest = (int)mlt;
+ *matchpos = matchPtr + back;
+ *startpos = ip + back;
}
- ipt += LZ4_NBCOMMONBYTES(diff);
- goto _endcount;
- }
- #if LZ4_ARCH64
- if ((ipt < (matchlimit - 3))
- && (A32(reft) == A32(ipt))) {
- ipt += 4;
- reft += 4;
}
- ipt += 2;
- #endif
- if ((ipt < (matchlimit - 1))
- && (A16(reft) == A16(ipt))) {
- reft += 2;
- }
- if ((ipt < matchlimit) && (*reft == *ipt))
- ipt++;
-_endcount:
- reft = ref;
-
- while ((startt > startlimit)
- && (reft > hc4->base)
- && (startt[-1] == reft[-1])) {
- startt--;
- reft--;
- }
-
- if ((ipt - startt) > longest) {
- longest = (int)(ipt - startt);
- *matchpos = reft;
- *startpos = startt;
+ }
+ } else {
+ const BYTE * const matchPtr = dictBase + matchIndex;
+
+ if (LZ4_read32(matchPtr) == LZ4_read32(ip)) {
+ size_t mlt;
+ int back = 0;
+ const BYTE *vLimit = ip + (dictLimit - matchIndex);
+
+ if (vLimit > iHighLimit)
+ vLimit = iHighLimit;
+
+ mlt = LZ4_count(ip + MINMATCH,
+ matchPtr + MINMATCH, vLimit) + MINMATCH;
+
+ if ((ip + mlt == vLimit) && (vLimit < iHighLimit))
+ mlt += LZ4_count(ip + mlt, base + dictLimit,
+ iHighLimit);
+ while ((ip + back > iLowLimit)
+ && (matchIndex + back > lowLimit)
+ && (ip[back - 1] == matchPtr[back - 1]))
+ back--;
+
+ mlt -= back;
+
+ if ((int)mlt > longest) {
+ longest = (int)mlt;
+ *matchpos = base + matchIndex + back;
+ *startpos = ip + back;
}
}
}
- ref -= (size_t)chaintable[(size_t)(ref) & MAXD_MASK];
+
+ matchIndex -= DELTANEXTU16(matchIndex);
}
+
return longest;
}
-static inline int lz4_encodesequence(const u8 **ip, u8 **op, const u8 **anchor,
- int ml, const u8 *ref)
+static inline int LZ4HC_encodeSequence(
+ const BYTE **ip,
+ BYTE **op,
+ const BYTE **anchor,
+ int matchLength,
+ const BYTE * const match,
+ limitedOutput_directive limitedOutputBuffer,
+ BYTE *oend)
{
- int length, len;
- u8 *token;
+ int length;
+ BYTE *token;
/* Encode Literal length */
length = (int)(*ip - *anchor);
token = (*op)++;
+
+ if ((limitedOutputBuffer)
+ && ((*op + (length>>8)
+ + length + (2 + 1 + LASTLITERALS)) > oend)) {
+ /* Check output limit */
+ return 1;
+ }
if (length >= (int)RUN_MASK) {
- *token = (RUN_MASK << ML_BITS);
+ int len;
+
+ *token = (RUN_MASK<<ML_BITS);
len = length - RUN_MASK;
for (; len > 254 ; len -= 255)
*(*op)++ = 255;
- *(*op)++ = (u8)len;
+ *(*op)++ = (BYTE)len;
} else
- *token = (length << ML_BITS);
+ *token = (BYTE)(length<<ML_BITS);
/* Copy Literals */
- LZ4_BLINDCOPY(*anchor, *op, length);
+ LZ4_wildCopy(*op, *anchor, (*op) + length);
+ *op += length;
/* Encode Offset */
- LZ4_WRITE_LITTLEENDIAN_16(*op, (u16)(*ip - ref));
+ LZ4_writeLE16(*op, (U16)(*ip - match));
+ *op += 2;
/* Encode MatchLength */
- len = (int)(ml - MINMATCH);
- if (len >= (int)ML_MASK) {
+ length = (int)(matchLength - MINMATCH);
+
+ if ((limitedOutputBuffer)
+ && (*op + (length>>8)
+ + (1 + LASTLITERALS) > oend)) {
+ /* Check output limit */
+ return 1;
+ }
+
+ if (length >= (int)ML_MASK) {
*token += ML_MASK;
- len -= ML_MASK;
- for (; len > 509 ; len -= 510) {
+ length -= ML_MASK;
+
+ for (; length > 509 ; length -= 510) {
*(*op)++ = 255;
*(*op)++ = 255;
}
- if (len > 254) {
- len -= 255;
+
+ if (length > 254) {
+ length -= 255;
*(*op)++ = 255;
}
- *(*op)++ = (u8)len;
+
+ *(*op)++ = (BYTE)length;
} else
- *token += len;
+ *token += (BYTE)(length);
/* Prepare next loop */
- *ip += ml;
+ *ip += matchLength;
*anchor = *ip;
return 0;
}
-static int lz4_compresshcctx(struct lz4hc_data *ctx,
- const char *source,
- char *dest,
- int isize)
+static int LZ4HC_compress_generic(
+ LZ4HC_CCtx_internal *const ctx,
+ const char * const source,
+ char * const dest,
+ int const inputSize,
+ int const maxOutputSize,
+ int compressionLevel,
+ limitedOutput_directive limit
+ )
{
- const u8 *ip = (const u8 *)source;
- const u8 *anchor = ip;
- const u8 *const iend = ip + isize;
- const u8 *const mflimit = iend - MFLIMIT;
- const u8 *const matchlimit = (iend - LASTLITERALS);
+ const BYTE *ip = (const BYTE *) source;
+ const BYTE *anchor = ip;
+ const BYTE * const iend = ip + inputSize;
+ const BYTE * const mflimit = iend - MFLIMIT;
+ const BYTE * const matchlimit = (iend - LASTLITERALS);
- u8 *op = (u8 *)dest;
+ BYTE *op = (BYTE *) dest;
+ BYTE * const oend = op + maxOutputSize;
+ unsigned int maxNbAttempts;
int ml, ml2, ml3, ml0;
- const u8 *ref = NULL;
- const u8 *start2 = NULL;
- const u8 *ref2 = NULL;
- const u8 *start3 = NULL;
- const u8 *ref3 = NULL;
- const u8 *start0;
- const u8 *ref0;
- int lastrun;
+ const BYTE *ref = NULL;
+ const BYTE *start2 = NULL;
+ const BYTE *ref2 = NULL;
+ const BYTE *start3 = NULL;
+ const BYTE *ref3 = NULL;
+ const BYTE *start0;
+ const BYTE *ref0;
+
+ /* init */
+ if (compressionLevel > LZ4HC_MAX_CLEVEL)
+ compressionLevel = LZ4HC_MAX_CLEVEL;
+ if (compressionLevel < 1)
+ compressionLevel = LZ4HC_DEFAULT_CLEVEL;
+ maxNbAttempts = 1 << (compressionLevel - 1);
+ ctx->end += inputSize;
ip++;
/* Main Loop */
while (ip < mflimit) {
- ml = lz4hc_insertandfindbestmatch(ctx, ip, matchlimit, (&ref));
+ ml = LZ4HC_InsertAndFindBestMatch(ctx, ip,
+ matchlimit, (&ref), maxNbAttempts);
if (!ml) {
ip++;
continue;
@@ -351,51 +387,59 @@ static int lz4_compresshcctx(struct lz4hc_data *ctx,
start0 = ip;
ref0 = ref;
ml0 = ml;
-_search2:
- if (ip+ml < mflimit)
- ml2 = lz4hc_insertandgetwidermatch(ctx, ip + ml - 2,
- ip + 1, matchlimit, ml, &ref2, &start2);
+
+_Search2:
+ if (ip + ml < mflimit)
+ ml2 = LZ4HC_InsertAndGetWiderMatch(ctx,
+ ip + ml - 2, ip + 0,
+ matchlimit, ml, &ref2,
+ &start2, maxNbAttempts);
else
ml2 = ml;
- /* No better match */
+
if (ml2 == ml) {
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
+ /* No better match */
+ if (LZ4HC_encodeSequence(&ip, &op,
+ &anchor, ml, ref, limit, oend))
+ return 0;
continue;
}
if (start0 < ip) {
- /* empirical */
if (start2 < ip + ml0) {
+ /* empirical */
ip = start0;
ref = ref0;
ml = ml0;
}
}
- /*
- * Here, start0==ip
- * First Match too small : removed
- */
+
+ /* Here, start0 == ip */
if ((start2 - ip) < 3) {
+ /* First Match too small : removed */
ml = ml2;
ip = start2;
ref = ref2;
- goto _search2;
+ goto _Search2;
}
-_search3:
+_Search3:
/*
- * Currently we have :
- * ml2 > ml1, and
- * ip1+3 <= ip2 (usually < ip1+ml1)
- */
+ * Currently we have :
+ * ml2 > ml1, and
+ * ip1 + 3 <= ip2 (usually < ip1 + ml1)
+ */
if ((start2 - ip) < OPTIMAL_ML) {
int correction;
int new_ml = ml;
+
if (new_ml > OPTIMAL_ML)
new_ml = OPTIMAL_ML;
if (ip + new_ml > start2 + ml2 - MINMATCH)
new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
+
correction = new_ml - (int)(start2 - ip);
+
if (correction > 0) {
start2 += correction;
ref2 += correction;
@@ -403,39 +447,44 @@ static int lz4_compresshcctx(struct lz4hc_data *ctx,
}
}
/*
- * Now, we have start2 = ip+new_ml,
- * with new_ml=min(ml, OPTIMAL_ML=18)
+ * Now, we have start2 = ip + new_ml,
+ * with new_ml = min(ml, OPTIMAL_ML = 18)
*/
+
if (start2 + ml2 < mflimit)
- ml3 = lz4hc_insertandgetwidermatch(ctx,
- start2 + ml2 - 3, start2, matchlimit,
- ml2, &ref3, &start3);
+ ml3 = LZ4HC_InsertAndGetWiderMatch(ctx,
+ start2 + ml2 - 3, start2,
+ matchlimit, ml2, &ref3, &start3,
+ maxNbAttempts);
else
ml3 = ml2;
- /* No better match : 2 sequences to encode */
if (ml3 == ml2) {
+ /* No better match : 2 sequences to encode */
/* ip & ref are known; Now for ml */
- if (start2 < ip+ml)
+ if (start2 < ip + ml)
ml = (int)(start2 - ip);
-
/* Now, encode 2 sequences */
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor,
+ ml, ref, limit, oend))
+ return 0;
ip = start2;
- lz4_encodesequence(&ip, &op, &anchor, ml2, ref2);
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor,
+ ml2, ref2, limit, oend))
+ return 0;
continue;
}
- /* Not enough space for match 2 : remove it */
if (start3 < ip + ml + 3) {
- /*
- * can write Seq1 immediately ==> Seq2 is removed,
- * so Seq3 becomes Seq1
- */
+ /* Not enough space for match 2 : remove it */
if (start3 >= (ip + ml)) {
+ /* can write Seq1 immediately
+ * ==> Seq2 is removed,
+ * so Seq3 becomes Seq1
+ */
if (start2 < ip + ml) {
- int correction =
- (int)(ip + ml - start2);
+ int correction = (int)(ip + ml - start2);
+
start2 += correction;
ref2 += correction;
ml2 -= correction;
@@ -446,35 +495,38 @@ static int lz4_compresshcctx(struct lz4hc_data *ctx,
}
}
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
- ip = start3;
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor,
+ ml, ref, limit, oend))
+ return 0;
+ ip = start3;
ref = ref3;
- ml = ml3;
+ ml = ml3;
start0 = start2;
ref0 = ref2;
ml0 = ml2;
- goto _search2;
+ goto _Search2;
}
start2 = start3;
ref2 = ref3;
ml2 = ml3;
- goto _search3;
+ goto _Search3;
}
/*
- * OK, now we have 3 ascending matches; let's write at least
- * the first one ip & ref are known; Now for ml
- */
+ * OK, now we have 3 ascending matches;
+ * let's write at least the first one
+ * ip & ref are known; Now for ml
+ */
if (start2 < ip + ml) {
if ((start2 - ip) < (int)ML_MASK) {
int correction;
+
if (ml > OPTIMAL_ML)
ml = OPTIMAL_ML;
if (ip + ml > start2 + ml2 - MINMATCH)
- ml = (int)(start2 - ip) + ml2
- - MINMATCH;
+ ml = (int)(start2 - ip) + ml2 - MINMATCH;
correction = ml - (int)(start2 - ip);
if (correction > 0) {
start2 += correction;
@@ -484,7 +536,9 @@ static int lz4_compresshcctx(struct lz4hc_data *ctx,
} else
ml = (int)(start2 - ip);
}
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
+ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml,
+ ref, limit, oend))
+ return 0;
ip = start2;
ref = ref2;
@@ -494,46 +548,245 @@ static int lz4_compresshcctx(struct lz4hc_data *ctx,
ref2 = ref3;
ml2 = ml3;
- goto _search3;
+ goto _Search3;
}
/* Encode Last Literals */
- lastrun = (int)(iend - anchor);
- if (lastrun >= (int)RUN_MASK) {
- *op++ = (RUN_MASK << ML_BITS);
- lastrun -= RUN_MASK;
- for (; lastrun > 254 ; lastrun -= 255)
- *op++ = 255;
- *op++ = (u8) lastrun;
- } else
- *op++ = (lastrun << ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend - anchor;
+ {
+ int lastRun = (int)(iend - anchor);
+
+ if ((limit)
+ && (((char *)op - dest) + lastRun + 1
+ + ((lastRun + 255 - RUN_MASK)/255)
+ > (U32)maxOutputSize)) {
+ /* Check output limit */
+ return 0;
+ }
+ if (lastRun >= (int)RUN_MASK) {
+ *op++ = (RUN_MASK<<ML_BITS);
+ lastRun -= RUN_MASK;
+ for (; lastRun > 254 ; lastRun -= 255)
+ *op++ = 255;
+ *op++ = (BYTE) lastRun;
+ } else
+ *op++ = (BYTE)(lastRun<<ML_BITS);
+ memcpy(op, anchor, iend - anchor);
+ op += iend - anchor;
+ }
+
/* End */
return (int) (((char *)op) - dest);
}
-int lz4hc_compress(const unsigned char *src, size_t src_len,
- unsigned char *dst, size_t *dst_len, void *wrkmem)
+static int LZ4_compress_HC_extStateHC(
+ void *state,
+ const char *src,
+ char *dst,
+ int srcSize,
+ int maxDstSize,
+ int compressionLevel)
{
- int ret = -1;
- int out_len = 0;
+ LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t *)state)->internal_donotuse;
- struct lz4hc_data *hc4 = (struct lz4hc_data *)wrkmem;
- lz4hc_init(hc4, (const u8 *)src);
- out_len = lz4_compresshcctx((struct lz4hc_data *)hc4, (const u8 *)src,
- (char *)dst, (int)src_len);
+ if (((size_t)(state)&(sizeof(void *) - 1)) != 0) {
+ /* Error : state is not aligned
+ * for pointers (32 or 64 bits)
+ */
+ return 0;
+ }
- if (out_len < 0)
- goto exit;
+ LZ4HC_init(ctx, (const BYTE *)src);
- *dst_len = out_len;
- return 0;
+ if (maxDstSize < LZ4_compressBound(srcSize))
+ return LZ4HC_compress_generic(ctx, src, dst,
+ srcSize, maxDstSize, compressionLevel, limitedOutput);
+ else
+ return LZ4HC_compress_generic(ctx, src, dst,
+ srcSize, maxDstSize, compressionLevel, noLimit);
+}
+
+int LZ4_compress_HC(const char *src, char *dst, int srcSize,
+ int maxDstSize, int compressionLevel, void *wrkmem)
+{
+ return LZ4_compress_HC_extStateHC(wrkmem, src, dst,
+ srcSize, maxDstSize, compressionLevel);
+}
+EXPORT_SYMBOL(LZ4_compress_HC);
+
+/**************************************
+ * Streaming Functions
+ **************************************/
+void LZ4_resetStreamHC(LZ4_streamHC_t *LZ4_streamHCPtr, int compressionLevel)
+{
+ LZ4_streamHCPtr->internal_donotuse.base = NULL;
+ LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned int)compressionLevel;
+}
+
+int LZ4_loadDictHC(LZ4_streamHC_t *LZ4_streamHCPtr,
+ const char *dictionary,
+ int dictSize)
+{
+ LZ4HC_CCtx_internal *ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+
+ if (dictSize > 64 * KB) {
+ dictionary += dictSize - 64 * KB;
+ dictSize = 64 * KB;
+ }
+ LZ4HC_init(ctxPtr, (const BYTE *)dictionary);
+ if (dictSize >= 4)
+ LZ4HC_Insert(ctxPtr, (const BYTE *)dictionary + (dictSize - 3));
+ ctxPtr->end = (const BYTE *)dictionary + dictSize;
+ return dictSize;
+}
+EXPORT_SYMBOL(LZ4_loadDictHC);
+
+/* compression */
+
+static void LZ4HC_setExternalDict(
+ LZ4HC_CCtx_internal *ctxPtr,
+ const BYTE *newBlock)
+{
+ if (ctxPtr->end >= ctxPtr->base + 4) {
+ /* Referencing remaining dictionary content */
+ LZ4HC_Insert(ctxPtr, ctxPtr->end - 3);
+ }
+
+ /*
+ * Only one memory segment for extDict,
+ * so any previous extDict is lost at this stage
+ */
+ ctxPtr->lowLimit = ctxPtr->dictLimit;
+ ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
+ ctxPtr->dictBase = ctxPtr->base;
+ ctxPtr->base = newBlock - ctxPtr->dictLimit;
+ ctxPtr->end = newBlock;
+ /* match referencing will resume from there */
+ ctxPtr->nextToUpdate = ctxPtr->dictLimit;
+}
+EXPORT_SYMBOL(LZ4HC_setExternalDict);
+
+static int LZ4_compressHC_continue_generic(
+ LZ4_streamHC_t *LZ4_streamHCPtr,
+ const char *source,
+ char *dest,
+ int inputSize,
+ int maxOutputSize,
+ limitedOutput_directive limit)
+{
+ LZ4HC_CCtx_internal *ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
+
+ /* auto - init if forgotten */
+ if (ctxPtr->base == NULL)
+ LZ4HC_init(ctxPtr, (const BYTE *) source);
+
+ /* Check overflow */
+ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 * GB) {
+ size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base)
+ - ctxPtr->dictLimit;
+ if (dictSize > 64 * KB)
+ dictSize = 64 * KB;
+ LZ4_loadDictHC(LZ4_streamHCPtr,
+ (const char *)(ctxPtr->end) - dictSize, (int)dictSize);
+ }
+
+ /* Check if blocks follow each other */
+ if ((const BYTE *)source != ctxPtr->end)
+ LZ4HC_setExternalDict(ctxPtr, (const BYTE *)source);
+
+ /* Check overlapping input/dictionary space */
+ {
+ const BYTE *sourceEnd = (const BYTE *) source + inputSize;
+ const BYTE * const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
+ const BYTE * const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
+
+ if ((sourceEnd > dictBegin)
+ && ((const BYTE *)source < dictEnd)) {
+ if (sourceEnd > dictEnd)
+ sourceEnd = dictEnd;
+ ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
+
+ if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4)
+ ctxPtr->lowLimit = ctxPtr->dictLimit;
+ }
+ }
+
+ return LZ4HC_compress_generic(ctxPtr, source, dest,
+ inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
+}
+
+int LZ4_compress_HC_continue(
+ LZ4_streamHC_t *LZ4_streamHCPtr,
+ const char *source,
+ char *dest,
+ int inputSize,
+ int maxOutputSize)
+{
+ if (maxOutputSize < LZ4_compressBound(inputSize))
+ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr,
+ source, dest, inputSize, maxOutputSize, limitedOutput);
+ else
+ return LZ4_compressHC_continue_generic(LZ4_streamHCPtr,
+ source, dest, inputSize, maxOutputSize, noLimit);
+}
+EXPORT_SYMBOL(LZ4_compress_HC_continue);
+
+/* dictionary saving */
+
+int LZ4_saveDictHC(
+ LZ4_streamHC_t *LZ4_streamHCPtr,
+ char *safeBuffer,
+ int dictSize)
+{
+ LZ4HC_CCtx_internal *const streamPtr = &LZ4_streamHCPtr->internal_donotuse;
+ int const prefixSize = (int)(streamPtr->end
+ - (streamPtr->base + streamPtr->dictLimit));
+
+ if (dictSize > 64 * KB)
+ dictSize = 64 * KB;
+ if (dictSize < 4)
+ dictSize = 0;
+ if (dictSize > prefixSize)
+ dictSize = prefixSize;
+
+ memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
-exit:
- return ret;
+ {
+ U32 const endIndex = (U32)(streamPtr->end - streamPtr->base);
+
+ streamPtr->end = (const BYTE *)safeBuffer + dictSize;
+ streamPtr->base = streamPtr->end - endIndex;
+ streamPtr->dictLimit = endIndex - dictSize;
+ streamPtr->lowLimit = endIndex - dictSize;
+
+ if (streamPtr->nextToUpdate < streamPtr->dictLimit)
+ streamPtr->nextToUpdate = streamPtr->dictLimit;
+ }
+ return dictSize;
+}
+EXPORT_SYMBOL(LZ4_saveDictHC);
+
+/*-******************************
+ * For backwards compatibility
+ ********************************/
+int lz4hc_compress(const unsigned char *src, size_t src_len,
+ unsigned char *dst, size_t *dst_len, void *wrkmem)
+{
+ *dst_len = LZ4_compress_HC(src, dst, src_len,
+ *dst_len, LZ4HC_DEFAULT_CLEVEL, wrkmem);
+
+ /*
+ * Prior lz4hc_compress will return -1 in case of error
+ * and 0 on success
+ * while new LZ4_compress_HC
+ * returns 0 in case of error
+ * and the output length on success
+ */
+ if (!*dst_len)
+ return -1;
+ else
+ return 0;
}
EXPORT_SYMBOL(lz4hc_compress);
MODULE_LICENSE("Dual BSD/GPL");
-MODULE_DESCRIPTION("LZ4HC compressor");
+MODULE_DESCRIPTION("LZ4 HC compressor");
--
2.1.4
^ permalink raw reply related
* [PATCH v3] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Ard Biesheuvel @ 2017-02-05 10:06 UTC (permalink / raw)
To: linux-crypto, ebiggers3; +Cc: herbert, Jason, Ard Biesheuvel
Instead of unconditionally forcing 4 byte alignment for all generic
chaining modes that rely on crypto_xor() or crypto_inc() (which may
result in unnecessary copying of data when the underlying hardware
can perform unaligned accesses efficiently), make those functions
deal with unaligned input explicitly, but only if the Kconfig symbol
HAVE_EFFICIENT_UNALIGNED_ACCESS is set. This will allow us to drop
the alignmasks from the CBC, CMAC, CTR, CTS, PCBC and SEQIV drivers.
For crypto_inc(), this simply involves making the 4-byte stride
conditional on HAVE_EFFICIENT_UNALIGNED_ACCESS being set, given that
it typically operates on 16 byte buffers.
For crypto_xor(), an algorithm is implemented that simply runs through
the input using the largest strides possible if unaligned accesses are
allowed. If they are not, an optimal sequence of memory accesses is
emitted that takes the relative alignment of the input buffers into
account, e.g., if the relative misalignment of dst and src is 4 bytes,
the entire xor operation will be completed using 4 byte loads and stores
(modulo unaligned bits at the start and end). Note that all expressions
involving misalign are simply eliminated by the compiler when
HAVE_EFFICIENT_UNALIGNED_ACCESS is defined.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
v3: fix thinko in processing of unaligned leading chunk
inline common case where the input size is a constant multiple of the word
size on architectures with h/w handling of unaligned accesses
crypto/algapi.c | 68 ++++++++++++++------
crypto/cbc.c | 3 -
crypto/cmac.c | 3 +-
crypto/ctr.c | 2 +-
crypto/cts.c | 3 -
crypto/pcbc.c | 3 -
crypto/seqiv.c | 2 -
include/crypto/algapi.h | 20 +++++-
8 files changed, 70 insertions(+), 34 deletions(-)
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 1fad2a6b3bbb..6b52e8f0b95f 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -962,34 +962,66 @@ void crypto_inc(u8 *a, unsigned int size)
__be32 *b = (__be32 *)(a + size);
u32 c;
- for (; size >= 4; size -= 4) {
- c = be32_to_cpu(*--b) + 1;
- *b = cpu_to_be32(c);
- if (c)
- return;
- }
+ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
+ !((unsigned long)b & (__alignof__(*b) - 1)))
+ for (; size >= 4; size -= 4) {
+ c = be32_to_cpu(*--b) + 1;
+ *b = cpu_to_be32(c);
+ if (c)
+ return;
+ }
crypto_inc_byte(a, size);
}
EXPORT_SYMBOL_GPL(crypto_inc);
-static inline void crypto_xor_byte(u8 *a, const u8 *b, unsigned int size)
+void __crypto_xor(u8 *dst, const u8 *src, unsigned int len)
{
- for (; size; size--)
- *a++ ^= *b++;
-}
+ int relalign = 0;
+
+ if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) {
+ int size = sizeof(unsigned long);
+ int d = ((unsigned long)dst ^ (unsigned long)src) & (size - 1);
+
+ relalign = d ? 1 << __ffs(d) : size;
+
+ /*
+ * If we care about alignment, process as many bytes as
+ * needed to advance dst and src to values whose alignments
+ * equal their relative alignment. This will allow us to
+ * process the remainder of the input using optimal strides.
+ */
+ while (((unsigned long)dst & (relalign - 1)) && len > 0) {
+ *dst++ ^= *src++;
+ len--;
+ }
+ }
-void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
-{
- u32 *a = (u32 *)dst;
- u32 *b = (u32 *)src;
+ while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) {
+ *(u64 *)dst ^= *(u64 *)src;
+ dst += 8;
+ src += 8;
+ len -= 8;
+ }
- for (; size >= 4; size -= 4)
- *a++ ^= *b++;
+ while (len >= 4 && !(relalign & 3)) {
+ *(u32 *)dst ^= *(u32 *)src;
+ dst += 4;
+ src += 4;
+ len -= 4;
+ }
+
+ while (len >= 2 && !(relalign & 1)) {
+ *(u16 *)dst ^= *(u16 *)src;
+ dst += 2;
+ src += 2;
+ len -= 2;
+ }
- crypto_xor_byte((u8 *)a, (u8 *)b, size);
+ while (len--)
+ *dst++ ^= *src++;
}
-EXPORT_SYMBOL_GPL(crypto_xor);
+EXPORT_SYMBOL_GPL(__crypto_xor);
unsigned int crypto_alg_extsize(struct crypto_alg *alg)
{
diff --git a/crypto/cbc.c b/crypto/cbc.c
index 68f751a41a84..bc160a3186dc 100644
--- a/crypto/cbc.c
+++ b/crypto/cbc.c
@@ -145,9 +145,6 @@ static int crypto_cbc_create(struct crypto_template *tmpl, struct rtattr **tb)
inst->alg.base.cra_blocksize = alg->cra_blocksize;
inst->alg.base.cra_alignmask = alg->cra_alignmask;
- /* We access the data as u32s when xoring. */
- inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
-
inst->alg.ivsize = alg->cra_blocksize;
inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
diff --git a/crypto/cmac.c b/crypto/cmac.c
index 04080dca8f0c..16301f52858c 100644
--- a/crypto/cmac.c
+++ b/crypto/cmac.c
@@ -260,8 +260,7 @@ static int cmac_create(struct crypto_template *tmpl, struct rtattr **tb)
if (err)
goto out_free_inst;
- /* We access the data as u32s when xoring. */
- alignmask = alg->cra_alignmask | (__alignof__(u32) - 1);
+ alignmask = alg->cra_alignmask;
inst->alg.base.cra_alignmask = alignmask;
inst->alg.base.cra_priority = alg->cra_priority;
inst->alg.base.cra_blocksize = alg->cra_blocksize;
diff --git a/crypto/ctr.c b/crypto/ctr.c
index a9a7a44f2783..a4f4a8983169 100644
--- a/crypto/ctr.c
+++ b/crypto/ctr.c
@@ -209,7 +209,7 @@ static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb)
inst->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
inst->alg.cra_priority = alg->cra_priority;
inst->alg.cra_blocksize = 1;
- inst->alg.cra_alignmask = alg->cra_alignmask | (__alignof__(u32) - 1);
+ inst->alg.cra_alignmask = alg->cra_alignmask;
inst->alg.cra_type = &crypto_blkcipher_type;
inst->alg.cra_blkcipher.ivsize = alg->cra_blocksize;
diff --git a/crypto/cts.c b/crypto/cts.c
index a1335d6c35fb..243f591dc409 100644
--- a/crypto/cts.c
+++ b/crypto/cts.c
@@ -374,9 +374,6 @@ static int crypto_cts_create(struct crypto_template *tmpl, struct rtattr **tb)
inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
- /* We access the data as u32s when xoring. */
- inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
-
inst->alg.ivsize = alg->base.cra_blocksize;
inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
diff --git a/crypto/pcbc.c b/crypto/pcbc.c
index 11d248673ad4..29dd2b4a3b85 100644
--- a/crypto/pcbc.c
+++ b/crypto/pcbc.c
@@ -260,9 +260,6 @@ static int crypto_pcbc_create(struct crypto_template *tmpl, struct rtattr **tb)
inst->alg.base.cra_blocksize = alg->cra_blocksize;
inst->alg.base.cra_alignmask = alg->cra_alignmask;
- /* We access the data as u32s when xoring. */
- inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
-
inst->alg.ivsize = alg->cra_blocksize;
inst->alg.min_keysize = alg->cra_cipher.cia_min_keysize;
inst->alg.max_keysize = alg->cra_cipher.cia_max_keysize;
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index c7049231861f..570b7d1aa0ca 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -153,8 +153,6 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb)
if (IS_ERR(inst))
return PTR_ERR(inst);
- inst->alg.base.cra_alignmask |= __alignof__(u32) - 1;
-
spawn = aead_instance_ctx(inst);
alg = crypto_spawn_aead_alg(spawn);
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 404e9558e879..ebe4ded0c55d 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -191,9 +191,25 @@ static inline unsigned int crypto_queue_len(struct crypto_queue *queue)
return queue->qlen;
}
-/* These functions require the input/output to be aligned as u32. */
void crypto_inc(u8 *a, unsigned int size);
-void crypto_xor(u8 *dst, const u8 *src, unsigned int size);
+void __crypto_xor(u8 *dst, const u8 *src, unsigned int size);
+
+static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
+{
+ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
+ __builtin_constant_p(size) &&
+ (size % sizeof(unsigned long)) == 0) {
+ unsigned long *d = (unsigned long *)dst;
+ unsigned long *s = (unsigned long *)src;
+
+ while (size > 0) {
+ *d++ ^= *s++;
+ size -= sizeof(unsigned long);
+ }
+ } else {
+ __crypto_xor(dst, src, size);
+ }
+}
int blkcipher_walk_done(struct blkcipher_desc *desc,
struct blkcipher_walk *walk, int err);
--
2.7.4
^ permalink raw reply related
* Re: [PATCH v2 4/6] crypto: ecdsa: add ECDSA SW implementation
From: Stephan Müller @ 2017-02-05 9:51 UTC (permalink / raw)
To: Nitin Kumbhar; +Cc: herbert, davem, linux-crypto
In-Reply-To: <1486120375-13070-5-git-send-email-nkumbhar@nvidia.com>
Am Freitag, 3. Februar 2017, 16:42:53 CET schrieb Nitin Kumbhar:
Hi Nitin,
> +
> +int ecdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key,
> + unsigned int keylen)
> +{
> + struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
> + struct ecdsa params;
> + unsigned int ndigits;
> + unsigned int nbytes;
> + u8 *params_qx, *params_qy;
> + u64 *ctx_qx, *ctx_qy;
> + int err = 0;
> +
> + if (crypto_ecdsa_parse_pub_key(key, keylen, ¶ms))
> + return -EINVAL;
> +
> + ndigits = ecdsa_supported_curve(params.curve_id);
> + if (!ndigits)
> + return -EINVAL;
> +
> + err = ecc_is_pub_key_valid(params.curve_id, ndigits,
> + params.key, params.key_size);
> + if (err)
> + return err;
> +
> + ctx->curve_id = params.curve_id;
> + ctx->ndigits = ndigits;
> + nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
> +
> + params_qx = params.key;
> + params_qy = params_qx + ECC_MAX_DIGIT_BYTES;
> +
> + ctx_qx = ctx->public_key;
> + ctx_qy = ctx_qx + ECC_MAX_DIGITS;
> +
> + vli_copy_from_buf(ctx_qx, ndigits, params_qx, nbytes);
> + vli_copy_from_buf(ctx_qy, ndigits, params_qy, nbytes);
> +
> + memset(¶ms, 0, sizeof(params));
> + return 0;
> +}
> +
> +int ecdsa_set_priv_key(struct crypto_akcipher *tfm, const void *key,
> + unsigned int keylen)
> +{
> + struct ecdsa_ctx *ctx = ecdsa_get_ctx(tfm);
> + struct ecdsa params;
> + unsigned int ndigits;
> + unsigned int nbytes;
> +
> + if (crypto_ecdsa_parse_priv_key(key, keylen, ¶ms))
> + return -EINVAL;
> +
> + ndigits = ecdsa_supported_curve(params.curve_id);
> + if (!ndigits)
> + return -EINVAL;
> +
> + ctx->curve_id = params.curve_id;
> + ctx->ndigits = ndigits;
> + nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
> +
> + if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits,
> + (const u8 *)params.key, params.key_size) < 0)
> + return -EINVAL;
> +
> + vli_copy_from_buf(ctx->private_key, ndigits, params.key, nbytes);
> +
> + memset(¶ms, 0, sizeof(params));
Please use memzero_explicit as otherwise this memset will be optimized away. I
think it could be used for the set_pub_key too, but there we do not have
sensitive data and thus it would not be strictly needed.
> + return 0;
> +}
Ciao
Stephan
^ permalink raw reply
* Re: [PATCH 5/6] dmaengine: Add Broadcom SBA RAID driver
From: Vinod Koul @ 2017-02-05 6:06 UTC (permalink / raw)
To: Anup Patel
Cc: Rob Herring, Mark Rutland, Herbert Xu, David S . Miller,
Jassi Brar, Dan Williams, Ray Jui, Scott Branden, Jon Mason,
Rob Rice, bcm-kernel-feedback-list-dY08KVG/lbpWk0Htik3J/w,
dmaengine-u79uwXL29TY76Z2rM5mHXA,
devicetree-u79uwXL29TY76Z2rM5mHXA,
linux-arm-kernel-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r,
linux-kernel-u79uwXL29TY76Z2rM5mHXA,
linux-crypto-u79uwXL29TY76Z2rM5mHXA,
linux-raid-u79uwXL29TY76Z2rM5mHXA
In-Reply-To: <1486010836-25228-6-git-send-email-anup.patel-dY08KVG/lbpWk0Htik3J/w@public.gmane.org>
On Thu, Feb 02, 2017 at 10:17:15AM +0530, Anup Patel wrote:
> +config BCM_SBA_RAID
> + tristate "Broadcom SBA RAID engine support"
> + depends on (ARM64 && MAILBOX && RAID6_PQ) || COMPILE_TEST
> + select DMA_ENGINE
> + select DMA_ENGINE_RAID
> + select ASYNC_TX_ENABLE_CHANNEL_SWITCH
> + default ARCH_BCM_IPROC
whats with the funny alignement?
> +/* SBA command related defines */
> +#define SBA_TYPE_SHIFT 48
> +#define SBA_TYPE_MASK 0x3
> +#define SBA_TYPE_A 0x0
> +#define SBA_TYPE_B 0x2
> +#define SBA_TYPE_C 0x3
> +#define SBA_USER_DEF_SHIFT 32
> +#define SBA_USER_DEF_MASK 0xffff
> +#define SBA_R_MDATA_SHIFT 24
> +#define SBA_R_MDATA_MASK 0xff
> +#define SBA_C_MDATA_MS_SHIFT 18
> +#define SBA_C_MDATA_MS_MASK 0x3
> +#define SBA_INT_SHIFT 17
> +#define SBA_INT_MASK 0x1
> +#define SBA_RESP_SHIFT 16
> +#define SBA_RESP_MASK 0x1
> +#define SBA_C_MDATA_SHIFT 8
> +#define SBA_C_MDATA_MASK 0xff
> +#define SBA_CMD_SHIFT 0
> +#define SBA_CMD_MASK 0xf
> +#define SBA_CMD_ZERO_ALL_BUFFERS 0x8
> +#define SBA_CMD_LOAD_BUFFER 0x9
> +#define SBA_CMD_XOR 0xa
> +#define SBA_CMD_GALOIS_XOR 0xb
> +#define SBA_CMD_ZERO_BUFFER 0x4
> +#define SBA_CMD_WRITE_BUFFER 0xc
Try using BIT and GENMAST for hardware descriptions
> +
> +/* SBA C_MDATA helper macros */
> +#define SBA_C_MDATA_LOAD_VAL(__bnum0) ((__bnum0) & 0x3)
> +#define SBA_C_MDATA_WRITE_VAL(__bnum0) ((__bnum0) & 0x3)
> +#define SBA_C_MDATA_XOR_VAL(__bnum1, __bnum0) \
> + ({ u32 __v = ((__bnum0) & 0x3); \
> + __v |= ((__bnum1) & 0x3) << 2; \
> + __v; \
> + })
> +#define SBA_C_MDATA_PQ_VAL(__dnum, __bnum1, __bnum0) \
> + ({ u32 __v = ((__bnum0) & 0x3); \
> + __v |= ((__bnum1) & 0x3) << 2; \
> + __v |= ((__dnum) & 0x1f) << 5; \
> + __v; \
> + })
ah why are we usig complex macros, why can't these be simple functions..
> +#define SBA_C_MDATA_LS(__c_mdata_val) ((__c_mdata_val) & 0xff)
> +#define SBA_C_MDATA_MS(__c_mdata_val) (((__c_mdata_val) >> 8) & 0x3)
> +
> +/* Driver helper macros */
> +#define to_sba_request(tx) \
> + container_of(tx, struct sba_request, tx)
> +#define to_sba_device(dchan) \
> + container_of(dchan, struct sba_device, dma_chan)
> +
> +enum sba_request_state {
> + SBA_REQUEST_STATE_FREE = 1,
> + SBA_REQUEST_STATE_ALLOCED = 2,
> + SBA_REQUEST_STATE_PENDING = 3,
> + SBA_REQUEST_STATE_ACTIVE = 4,
> + SBA_REQUEST_STATE_COMPLETED = 5,
> + SBA_REQUEST_STATE_ABORTED = 6,
whats up with a very funny indentation setting, we use 8 chars.
Please re-read the Documentation/process/coding-style.rst
> +static int sba_alloc_chan_resources(struct dma_chan *dchan)
> +{
> + /*
> + * We only have one channel so we have pre-alloced
> + * channel resources. Over here we just return number
> + * of free request.
> + */
> + return sba_free_request_count(to_sba_device(dchan));
> +}
essentially you are not doing much, so you can skip it. Its an optional
call.
> +static void sba_free_chan_resources(struct dma_chan *dchan)
> +{
> + /*
> + * Channel resources are pre-alloced so we just free-up
> + * whatever we can so that we can re-use pre-alloced
> + * channel resources next time.
> + */
> + sba_cleanup_inflight_requests(to_sba_device(dchan));
well this one checks for pending requests as well, which shouldn't be there
when freeing a channel, something seems not quite right here..
> +static int sba_send_mbox_request(struct sba_device *sba,
> + struct sba_request *req)
> +{
> + int mchans_idx, ret = 0;
> +
> + /* Select mailbox channel in round-robin fashion */
> + mchans_idx = atomic_inc_return(&sba->mchans_current);
> + mchans_idx = mchans_idx % sba->mchans_count;
> +
> + /* Send batch message for the request */
> + req->bmsg.batch.msgs_queued = 0;
> + ret = mbox_send_message(sba->mchans[mchans_idx], &req->bmsg);
> + if (ret < 0) {
> + dev_info(sba->dev, "channel %d message %d (total %d)",
> + mchans_idx, req->bmsg.batch.msgs_queued,
> + req->bmsg.batch.msgs_count);
dev_err?
> + dev_err(sba->dev, "send message failed with error %d", ret);
> + return ret;
> + }
> + ret = req->bmsg.error;
> + if (ret < 0) {
> + dev_info(sba->dev,
> + "mbox channel %d message %d (total %d)",
> + mchans_idx, req->bmsg.batch.msgs_queued,
> + req->bmsg.batch.msgs_count);
same here
> +static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx)
> +{
> + unsigned long flags;
> + dma_cookie_t cookie;
> + struct sba_request *req;
> + struct sba_device *sba;
> +
> + if (unlikely(!tx))
> + return -EINVAL;
> +
> + sba = to_sba_device(tx->chan);
> + req = to_sba_request(tx);
> +
> + /* Assign cookie and mark request pending */
> + spin_lock_irqsave(&sba->reqs_lock, flags);
> + cookie = dma_cookie_assign(tx);
> + _sba_pending_request(sba, req);
> + spin_unlock_irqrestore(&sba->reqs_lock, flags);
> +
> + /* Try to submit pending request */
> + sba_issue_pending(&sba->dma_chan);
Nope, thats wrong, caller needs to call .issue_pending for that
> +static enum dma_status sba_tx_status(struct dma_chan *dchan,
> + dma_cookie_t cookie,
> + struct dma_tx_state *txstate)
> +{
> + int mchan_idx;
> + enum dma_status ret;
> + struct sba_device *sba = to_sba_device(dchan);
> +
> + ret = dma_cookie_status(dchan, cookie, txstate);
> + if (ret == DMA_COMPLETE)
> + return ret;
> +
> + for (mchan_idx = 0; mchan_idx < sba->mchans_count; mchan_idx++)
> + mbox_client_peek_data(sba->mchans[mchan_idx]);
what is this achieving?
> +static struct dma_async_tx_descriptor *
> +sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
> + size_t len, unsigned long flags)
> +{
> + size_t msg_len;
> + dma_addr_t msg_offset = 0;
> + unsigned int msgs_count = 0, cmds_count, cmds_idx = 0;
> + struct sba_device *sba = to_sba_device(dchan);
> + struct sba_request *req = NULL;
> +
> + /* Sanity checks */
> + if (unlikely(len > sba->req_size))
> + return NULL;
why is that an error, you can create multiple txn of max length
> +static int sba_async_register(struct sba_device *sba)
> +{
> + int ret;
> + struct dma_device *dma_dev = &sba->dma_dev;
> +
> + /* Initialize DMA channel cookie */
> + sba->dma_chan.device = dma_dev;
> + dma_cookie_init(&sba->dma_chan);
> +
> + /* Initialize DMA device capability mask */
> + dma_cap_zero(dma_dev->cap_mask);
> + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
> + dma_cap_set(DMA_XOR, dma_dev->cap_mask);
> + dma_cap_set(DMA_PQ, dma_dev->cap_mask);
> +
> + /*
> + * Set mailbox channel device as the base device of
> + * our dma_device because the actual memory accesses
> + * will be done by mailbox controller
> + */
> + dma_dev->dev = sba->mbox_dev;
> +
> + /* Set base prep routines */
> + dma_dev->device_alloc_chan_resources = sba_alloc_chan_resources;
> + dma_dev->device_free_chan_resources = sba_free_chan_resources;
> + dma_dev->device_issue_pending = sba_issue_pending;
> + dma_dev->device_tx_status = sba_tx_status;
Please add terminate callback support, also add the capabilities, we need to
advertise that and use in clients
Also you can simplify bunch of code by using virt-chan support for managing
channels and descriptors
--
~Vinod
--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [RFC PATCH] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Eric Biggers @ 2017-02-05 3:05 UTC (permalink / raw)
To: Jason A. Donenfeld; +Cc: Ard Biesheuvel, Linux Crypto Mailing List, Herbert Xu
In-Reply-To: <CAHmME9pwhWrXLEXyrPKueFwoVexEVNvSxN=FDFLsi+gHowE2Ww@mail.gmail.com>
On Sun, Feb 05, 2017 at 12:10:53AM +0100, Jason A. Donenfeld wrote:
> Another thing that might be helpful is that you can let gcc decide on
> the alignment, and then optimize appropriately. Check out what we do
> with siphash:
>
> https://git.kernel.org/cgit/linux/kernel/git/davem/net-next.git/tree/include/linux/siphash.h#n76
>
> static inline u64 siphash(const void *data, size_t len, const
> siphash_key_t *key)
> {
> #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
> if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
> return __siphash_unaligned(data, len, key);
> #endif
> return ___siphash_aligned(data, len, key);
> }
>
> With this trick, we fall through to the fast alignment-assuming code,
> if gcc can prove that the address is inlined. This is often the case
> when passing structs, or when passing buffers that have
> __aligned(BLOCKSIZE). It proves to be a very useful optimization on
> some platforms.
Yes, this is a good idea. Though it seems that usually at least one of the two
pointers passed to crypto_xor() will have alignment unknown to the compiler,
sometimes the length is constant which inlining can help a lot for. For
example, if someone does crypto_xor(foo, bar, 16) on x86_64 or ARM64, we'd
really like it to turn into just a few instructions like this:
mov (%rsi),%rax
xor %rax,(%rdi)
mov 0x8(%rsi),%rax
xor %rax,0x8(%rdi)
So how about inlining crypto_xor() if CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS or
the pointers are long-aligned, otherwise calling an out-of-line function
__crypto_xor_unaligned() that handles all the cases with weird alignment.
Something like the following patch:
(Note: exactly how __crypto_xor_unaligned() is implemented is still debatable;
it could be more similar to Ard's proposal, or it could use the unaligned access
helpers.)
diff --git a/crypto/algapi.c b/crypto/algapi.c
index df939b54b09f..a0591db3f13a 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -972,23 +972,69 @@ void crypto_inc(u8 *a, unsigned int size)
}
EXPORT_SYMBOL_GPL(crypto_inc);
-static inline void crypto_xor_byte(u8 *a, const u8 *b, unsigned int size)
+#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
+void __crypto_xor_unaligned(u8 *dst, const u8 *src, unsigned int len)
{
- for (; size; size--)
- *a++ ^= *b++;
-}
+ unsigned long delta = (unsigned long)dst ^ (unsigned long)src;
-void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
-{
- u32 *a = (u32 *)dst;
- u32 *b = (u32 *)src;
+ /* Handle relative misalignment */
+ if (delta % sizeof(unsigned long)) {
+
+ /* 1-byte relative misalignment? */
+ if (delta & 1) {
+ while (len--)
+ *dst++ ^= *src++;
+ return;
+ }
- for (; size >= 4; size -= 4)
- *a++ ^= *b++;
+ /* 2-byte relative misalignment? */
+ if ((delta & 2) || sizeof(unsigned long) == 4) {
+ if ((unsigned long)dst % __alignof__(u16) && len) {
+ *dst++ ^= *src++;
+ len--;
+ }
+ while (len >= 2) {
+ *(u16 *)dst ^= *(u16 *)src;
+ dst += 2, src += 2, len -= 2;
+ }
+ if (len)
+ *dst ^= *src;
+ return;
+ }
+
+ /* 4-byte relative misalignment? */
+ while ((unsigned long)dst % __alignof__(u32) && len) {
+ *dst++ ^= *src++;
+ len--;
+ }
+ while (len >= 4) {
+ *(u32 *)dst ^= *(u32 *)src;
+ dst += 4, src += 4, len -= 4;
+ }
+ while (len--)
+ *dst++ ^= *src++;
+ return;
+ }
+
+ /* No relative misalignment; use word accesses */
+
+ while ((unsigned long)dst % __alignof__(unsigned long) && len) {
+ *dst++ ^= *src++;
+ len--;
+ }
+
+ while (len >= sizeof(unsigned long)) {
+ *(unsigned long *)dst ^= *(unsigned long *)src;
+ dst += sizeof(unsigned long);
+ src += sizeof(unsigned long);
+ len -= sizeof(unsigned long);
+ }
- crypto_xor_byte((u8 *)a, (u8 *)b, size);
+ while (len--)
+ *dst++ ^= *src++;
}
-EXPORT_SYMBOL_GPL(crypto_xor);
+EXPORT_SYMBOL_GPL(__crypto_xor_unaligned);
+#endif /* !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
unsigned int crypto_alg_extsize(struct crypto_alg *alg)
{
diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h
index 404e9558e879..718145c5eaca 100644
--- a/include/crypto/algapi.h
+++ b/include/crypto/algapi.h
@@ -191,9 +191,29 @@ static inline unsigned int crypto_queue_len(struct crypto_queue *queue)
return queue->qlen;
}
-/* These functions require the input/output to be aligned as u32. */
void crypto_inc(u8 *a, unsigned int size);
-void crypto_xor(u8 *dst, const u8 *src, unsigned int size);
+
+void __crypto_xor_unaligned(u8 *dst, const u8 *src, unsigned int len);
+
+static inline void crypto_xor(u8 *dst, const u8 *src, unsigned int len)
+{
+ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
+ (((unsigned long)dst | (unsigned long)src) %
+ __alignof__(unsigned long) == 0))
+ {
+ while (len >= sizeof(unsigned long)) {
+ *(unsigned long *)dst ^= *(unsigned long *)src;
+ dst += sizeof(unsigned long);
+ src += sizeof(unsigned long);
+ len -= sizeof(unsigned long);
+ }
+
+ while (len--)
+ *dst++ ^= *src++;
+ return;
+ }
+ return __crypto_xor_unaligned(dst, src, len);
+}
int blkcipher_walk_done(struct blkcipher_desc *desc,
struct blkcipher_walk *walk, int err);
^ permalink raw reply related
* Re: [RFC PATCH] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Jason A. Donenfeld @ 2017-02-04 23:10 UTC (permalink / raw)
To: Eric Biggers; +Cc: Ard Biesheuvel, Linux Crypto Mailing List, Herbert Xu
In-Reply-To: <CAHmME9qtX64rp-bXfb7-vM-vsBd18186MNZ4G9KonLYhsM5uEg@mail.gmail.com>
Another thing that might be helpful is that you can let gcc decide on
the alignment, and then optimize appropriately. Check out what we do
with siphash:
https://git.kernel.org/cgit/linux/kernel/git/davem/net-next.git/tree/include/linux/siphash.h#n76
static inline u64 siphash(const void *data, size_t len, const
siphash_key_t *key)
{
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT))
return __siphash_unaligned(data, len, key);
#endif
return ___siphash_aligned(data, len, key);
}
With this trick, we fall through to the fast alignment-assuming code,
if gcc can prove that the address is inlined. This is often the case
when passing structs, or when passing buffers that have
__aligned(BLOCKSIZE). It proves to be a very useful optimization on
some platforms.
^ permalink raw reply
* Re: [RFC PATCH] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Jason A. Donenfeld @ 2017-02-04 23:00 UTC (permalink / raw)
To: Eric Biggers; +Cc: Ard Biesheuvel, Linux Crypto Mailing List, Herbert Xu
In-Reply-To: <20170202064716.GB582@zzz>
Hey,
On Thu, Feb 2, 2017 at 7:47 AM, Eric Biggers <ebiggers3@gmail.com> wrote:
> I'm wondering whether it has to be that way, especially since it seems to most
> commonly be used on very small input buffers, e.g. 8 or 16-byte blocks.
Note that popular stream ciphers like chacha or salsa wind up XORing
much longer blocks -- 64 bytes. Likewise, CTR mode tends to XOR using
the block size as well. Not sure whether this is directly relavent for
the decision making here, but I thought I'd mention it just in case.
The XOR for this case should be _fast_, and preferably inlineable.
Jason
^ permalink raw reply
* Re: [PATCH v2] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Ard Biesheuvel @ 2017-02-04 21:49 UTC (permalink / raw)
To: Eric Biggers; +Cc: linux-crypto@vger.kernel.org, Herbert Xu
In-Reply-To: <20170204212038.GA5621@zzz>
On 4 February 2017 at 21:20, Eric Biggers <ebiggers3@gmail.com> wrote:
> Hi Ard,
>
> On Thu, Feb 02, 2017 at 03:56:28PM +0000, Ard Biesheuvel wrote:
>> + const int size = sizeof(unsigned long);
>> + int delta = ((unsigned long)dst ^ (unsigned long)src) & (size - 1);
>> + int misalign = 0;
>> +
>> + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && delta) {
>> + misalign = 1 << __ffs(delta);
>> +
>> + /*
>> + * If we care about alignment, process as many bytes as
>> + * needed to advance dst and src to values whose alignments
>> + * equal their relative misalignment. This will allow us to
>> + * process the remainder of the input using optimal strides.
>> + */
>> + while (((unsigned long)dst & (misalign - 1)) && len > 0) {
>> + *dst++ ^= *src++;
>> + len--;
>> + }
>> + }
>>
>> + while (len >= size && !misalign) {
>> + *(unsigned long *)dst ^= *(unsigned long *)src;
>> + dst += size;
>> + src += size;
>> + len -= size;
>> + }
>>
>
> Unfortunately this is still broken, for two different reasons. First, if the
> pointers have the same relative misalignment, then 'delta' and 'misalign' will
> be set to 0 and long accesses will be used, even though the pointers may
> actually be misaligned, e.g. 0x80000001 and 0x90000001.
In this case, the initial loop should run 7 times, but it obviously does not :-(
> Second, if the pointers
> have a relative misalignent that is not a power-of-2, then 'misalign' will be
> set to the wrong value. For example, with delta=3, it's actually only safe to
> do byte accesses, but the code will set misalign=2 and do u16 accesses.
>
As you realised, delta == 3 results in misalign == 1, which I think
does the right thing (module the initial loop which is incorrect)
> I kind of liked the version with put_unaligned/get_unaligned (and it seems to
> perform okay on MIPS, though not on ARM which is probably more important).
The reason I don't like the _unaligned() accessors is because they
take the performance hit regardless of whether the pointer is aligned
or not.
> But
> if the various cases with !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS are going to
> be handled/optimized I think they will just need to be separated out, maybe
> something like this:
>
> void crypto_xor(u8 *dst, const u8 *src, unsigned int len)
> {
> #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
> unsigned long delta;
>
> if (((unsigned long)dst | (unsigned long)src | len) %
> sizeof(unsigned long) == 0) {
> /* fast path: everything is aligned, including len */
> while (len >= sizeof(unsigned long)) {
> *(unsigned long *)dst ^= *(unsigned long *)src;
> dst += sizeof(unsigned long);
> src += sizeof(unsigned long);
> len -= sizeof(unsigned long);
> }
> return;
> }
>
> /* handle relative misalignment */
> delta = (unsigned long)dst ^ (unsigned long)src;
> if (delta % sizeof(unsigned long)) {
>
> /* 1-byte relative misalignment: do byte accesses */
> if (delta & 1) {
> while (len--)
> *dst++ ^= *src++;
> return;
> }
>
> /* 2-byte relative misalignment: do u16 accesses */
> if ((delta & 2) || sizeof(unsigned long) == 4) {
> if ((unsigned long)dst % 2 && len) {
> *dst++ ^= *src++;
> len--;
> }
> while (len >= 2) {
> *(u16 *)dst ^= *(u16 *)src;
> dst += 2, src += 2, len -= 2;
> }
> if (len)
> *dst ^= *src;
> return;
> }
>
> /* 4-byte relative misalignment: do u32 accesses */
> while ((unsigned long)dst % 4 && len) {
> *dst++ ^= *src++;
> len--;
> }
> while (len >= 4) {
> *(u32 *)dst ^= *(u32 *)src;
> dst += 4, src += 4, len -= 4;
> }
> while (len--)
> *dst++ ^= *src++;
> return;
> }
>
> /* handle absolute misalignment */
> while ((unsigned long)dst % sizeof(unsigned long) && len) {
> *dst++ ^= *src++;
> len--;
> }
> #endif /* !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
>
> while (len >= sizeof(unsigned long)) {
> *(unsigned long *)dst ^= *(unsigned long *)src;
> dst += sizeof(unsigned long);
> src += sizeof(unsigned long);
> len -= sizeof(unsigned long);
> }
>
> while (len--)
> *dst++ ^= *src++;
> }
^ permalink raw reply
* Re: [PATCH v2] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Eric Biggers @ 2017-02-04 21:27 UTC (permalink / raw)
To: Ard Biesheuvel; +Cc: linux-crypto, herbert
In-Reply-To: <20170204212038.GA5621@zzz>
On Sat, Feb 04, 2017 at 01:20:38PM -0800, Eric Biggers wrote:
> Unfortunately this is still broken, for two different reasons. First, if the
> pointers have the same relative misalignment, then 'delta' and 'misalign' will
> be set to 0 and long accesses will be used, even though the pointers may
> actually be misaligned, e.g. 0x80000001 and 0x90000001. Second, if the pointers
> have a relative misalignent that is not a power-of-2, then 'misalign' will be
> set to the wrong value. For example, with delta=3, it's actually only safe to
> do byte accesses, but the code will set misalign=2 and do u16 accesses.
>
Correction: for the second issue I think I mixed up ffs and fls, so that part of
the code was right. But it may still be a good idea to separate out the
different cases.
Eric
^ permalink raw reply
* Re: [PATCH v2] crypto: algapi - make crypto_xor() and crypto_inc() alignment agnostic
From: Eric Biggers @ 2017-02-04 21:20 UTC (permalink / raw)
To: Ard Biesheuvel; +Cc: linux-crypto, herbert
In-Reply-To: <1486050988-20407-1-git-send-email-ard.biesheuvel@linaro.org>
Hi Ard,
On Thu, Feb 02, 2017 at 03:56:28PM +0000, Ard Biesheuvel wrote:
> + const int size = sizeof(unsigned long);
> + int delta = ((unsigned long)dst ^ (unsigned long)src) & (size - 1);
> + int misalign = 0;
> +
> + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && delta) {
> + misalign = 1 << __ffs(delta);
> +
> + /*
> + * If we care about alignment, process as many bytes as
> + * needed to advance dst and src to values whose alignments
> + * equal their relative misalignment. This will allow us to
> + * process the remainder of the input using optimal strides.
> + */
> + while (((unsigned long)dst & (misalign - 1)) && len > 0) {
> + *dst++ ^= *src++;
> + len--;
> + }
> + }
>
> + while (len >= size && !misalign) {
> + *(unsigned long *)dst ^= *(unsigned long *)src;
> + dst += size;
> + src += size;
> + len -= size;
> + }
>
Unfortunately this is still broken, for two different reasons. First, if the
pointers have the same relative misalignment, then 'delta' and 'misalign' will
be set to 0 and long accesses will be used, even though the pointers may
actually be misaligned, e.g. 0x80000001 and 0x90000001. Second, if the pointers
have a relative misalignent that is not a power-of-2, then 'misalign' will be
set to the wrong value. For example, with delta=3, it's actually only safe to
do byte accesses, but the code will set misalign=2 and do u16 accesses.
I kind of liked the version with put_unaligned/get_unaligned (and it seems to
perform okay on MIPS, though not on ARM which is probably more important). But
if the various cases with !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS are going to
be handled/optimized I think they will just need to be separated out, maybe
something like this:
void crypto_xor(u8 *dst, const u8 *src, unsigned int len)
{
#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
unsigned long delta;
if (((unsigned long)dst | (unsigned long)src | len) %
sizeof(unsigned long) == 0) {
/* fast path: everything is aligned, including len */
while (len >= sizeof(unsigned long)) {
*(unsigned long *)dst ^= *(unsigned long *)src;
dst += sizeof(unsigned long);
src += sizeof(unsigned long);
len -= sizeof(unsigned long);
}
return;
}
/* handle relative misalignment */
delta = (unsigned long)dst ^ (unsigned long)src;
if (delta % sizeof(unsigned long)) {
/* 1-byte relative misalignment: do byte accesses */
if (delta & 1) {
while (len--)
*dst++ ^= *src++;
return;
}
/* 2-byte relative misalignment: do u16 accesses */
if ((delta & 2) || sizeof(unsigned long) == 4) {
if ((unsigned long)dst % 2 && len) {
*dst++ ^= *src++;
len--;
}
while (len >= 2) {
*(u16 *)dst ^= *(u16 *)src;
dst += 2, src += 2, len -= 2;
}
if (len)
*dst ^= *src;
return;
}
/* 4-byte relative misalignment: do u32 accesses */
while ((unsigned long)dst % 4 && len) {
*dst++ ^= *src++;
len--;
}
while (len >= 4) {
*(u32 *)dst ^= *(u32 *)src;
dst += 4, src += 4, len -= 4;
}
while (len--)
*dst++ ^= *src++;
return;
}
/* handle absolute misalignment */
while ((unsigned long)dst % sizeof(unsigned long) && len) {
*dst++ ^= *src++;
len--;
}
#endif /* !CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */
while (len >= sizeof(unsigned long)) {
*(unsigned long *)dst ^= *(unsigned long *)src;
dst += sizeof(unsigned long);
src += sizeof(unsigned long);
len -= sizeof(unsigned long);
}
while (len--)
*dst++ ^= *src++;
}
^ permalink raw reply
* Re: [PATCH resend 4.9] hw_random: Don't use a stack buffer in add_early_randomness()
From: Yisheng Xie @ 2017-02-04 10:32 UTC (permalink / raw)
To: Andy Lutomirski, linux-crypto, linux-kernel, Matt Mackall,
Herbert Xu, Rusty Russell, Jens Axboe, Matt Mullins, Xishi Qiu,
Hanjun Guo
In-Reply-To: <20170204043416.GA7458@hydra.tuxags.com>
hi, Matt,
Thanks for your reply.
On 2017/2/4 12:34, Matt Mullins wrote:
> On Sat, Feb 04, 2017 at 11:47:38AM +0800, Yisheng Xie wrote:
>> On 2016/10/18 1:06, Andy Lutomirski wrote:
>>> hw_random carefully avoids using a stack buffer except in
>>> add_early_randomness(). This causes a crash in virtio_rng if
>>> CONFIG_VMAP_STACK=y.
>> I try to understand this patch, but I do not know why it will cause
>> a crash in virtio_rng with CONFIG_VMAP_STACK=y?
>> Could you please give me more info. about it.
>
> My original report was
> https://lkml.kernel.org/r/20161016002151.GA18235@hydra.tuxags.com.
>
> The virtio ring APIs use scatterlists to keep track of the buffers, and
> scatterlist requires addresses to be in the kernel direct-mapped address range.
> This is not the case for vmalloc()ed addresses, such as the original on-stack
> "bytes" array when VMAP_STACK=y.
>
I see, and will check the logic to get more detail about it.
Thanks.
Yisheng Xie
^ permalink raw reply
* Re: [PATCH v5 0/3] Add Support for Cavium Cryptographic Acceleration Unit
From: George Cherian @ 2017-02-04 5:17 UTC (permalink / raw)
To: Sasha Levin, George Cherian, alexander.levin
Cc: Herbert Xu, davem, david.daney, corentin labbe, smueller,
linux-kernel@vger.kernel.org List, linux-crypto
In-Reply-To: <CA+1xoqcWSdKtqvEQaGU+RQmxY3ZsH6L9G8_yFUODGcDksLuTiw@mail.gmail.com>
[-- Attachment #1: Type: text/plain, Size: 626 bytes --]
On Friday 03 February 2017 11:02 PM, Sasha Levin wrote:
> On Mon, Jan 30, 2017 at 7:30 AM, George Cherian
> <george.cherian@cavium.com> wrote:
>> This series adds the support for Cavium Cryptographic Accelerarion Unit (CPT)
>> CPT is available in Cavium's Octeon-Tx SoC series.
>>
>> The series was tested with ecryptfs and dm-crypt for in kernel cryptographic
>> offload operations. This driver needs a firmware to work, I will be sending the
>> firmware to linux-firmware once the driver is accepted.
> Can we have the firmware now to be able to actually test this series?
attached, Hope you have access to an 81xx board!!
[-- Attachment #2: cpt8x-mc-ae.out --]
[-- Type: application/octet-stream, Size: 10080 bytes --]
[-- Attachment #3: cpt8x-mc-se.out --]
[-- Type: application/octet-stream, Size: 34624 bytes --]
^ permalink raw reply
* Re: [PATCH resend 4.9] hw_random: Don't use a stack buffer in add_early_randomness()
From: Matt Mullins @ 2017-02-04 4:34 UTC (permalink / raw)
To: Yisheng Xie
Cc: Andy Lutomirski, linux-crypto, linux-kernel, Matt Mackall,
Herbert Xu, Rusty Russell, Jens Axboe, Matt Mullins, Xishi Qiu,
Hanjun Guo
In-Reply-To: <b0545f55-3b3c-35af-c9f6-bc63cf169d2d@huawei.com>
On Sat, Feb 04, 2017 at 11:47:38AM +0800, Yisheng Xie wrote:
> On 2016/10/18 1:06, Andy Lutomirski wrote:
> > hw_random carefully avoids using a stack buffer except in
> > add_early_randomness(). This causes a crash in virtio_rng if
> > CONFIG_VMAP_STACK=y.
> I try to understand this patch, but I do not know why it will cause
> a crash in virtio_rng with CONFIG_VMAP_STACK=y?
> Could you please give me more info. about it.
My original report was
https://lkml.kernel.org/r/20161016002151.GA18235@hydra.tuxags.com.
The virtio ring APIs use scatterlists to keep track of the buffers, and
scatterlist requires addresses to be in the kernel direct-mapped address range.
This is not the case for vmalloc()ed addresses, such as the original on-stack
"bytes" array when VMAP_STACK=y.
^ permalink raw reply
* Re: [PATCH v5 2/3] drivers: crypto: Add the Virtual Function driver for CPT
From: George Cherian @ 2017-02-04 4:37 UTC (permalink / raw)
To: Sasha Levin, George Cherian
Cc: Herbert Xu, davem, david.daney, clabbe.montjoie, smueller,
linux-kernel@vger.kernel.org List, linux-crypto, alexander.levin
In-Reply-To: <CA+1xoqfewuW2F26ZsJV6HHuDx3z1eP=0fVK_DsMysy6ntLL+SA@mail.gmail.com>
Hi Sasha,
Thanks for the reveiw.
On Friday 03 February 2017 12:24 AM, Sasha Levin wrote:
> On Mon, Jan 30, 2017 at 7:30 AM, George Cherian
> <george.cherian@cavium.com> wrote:
>> diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
>> new file mode 100644
>> index 0000000..4cf466d
>> --- /dev/null
>> +++ b/drivers/crypto/cavium/cpt/cptvf_main.c
>> @@ -0,0 +1,948 @@
>> +/*
>> + * Copyright (C) 2016 Cavium, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of version 2 of the GNU General Public License
>> + * as published by the Free Software Foundation.
>> + */
>> +
>> +#include <linux/interrupt.h>
>> +#include <linux/module.h>
>> +
>> +#include "cptvf.h"
>> +
>> +#define DRV_NAME "thunder-cptvf"
>> +#define DRV_VERSION "1.0"
>> +
>> +struct cptvf_wqe {
>> + struct tasklet_struct twork;
>> + void *cptvf;
>> + u32 qno;
>> +};
>> +
>> +struct cptvf_wqe_info {
>> + struct cptvf_wqe vq_wqe[CPT_NUM_QS_PER_VF];
>> +};
>> +
>> +static void vq_work_handler(unsigned long data)
>> +{
>> + struct cptvf_wqe_info *cwqe_info = (struct cptvf_wqe_info *)data;
>> + struct cptvf_wqe *cwqe = &cwqe_info->vq_wqe[0];
>> +
>> + vq_post_process(cwqe->cptvf, cwqe->qno);
>> +}
>> +
>> +static int init_worker_threads(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cptvf_wqe_info *cwqe_info;
>> + int i;
>> +
>> + cwqe_info = kzalloc(sizeof(*cwqe_info), GFP_KERNEL);
>> + if (!cwqe_info)
>> + return -ENOMEM;
>> +
>> + if (cptvf->nr_queues) {
>> + dev_info(&pdev->dev, "Creating VQ worker threads (%d)\n",
>> + cptvf->nr_queues);
>> + }
>> +
>> + for (i = 0; i < cptvf->nr_queues; i++) {
>> + tasklet_init(&cwqe_info->vq_wqe[i].twork, vq_work_handler,
>> + (u64)cwqe_info);
>> + cwqe_info->vq_wqe[i].qno = i;
>> + cwqe_info->vq_wqe[i].cptvf = cptvf;
>> + }
>> +
>> + cptvf->wqe_info = cwqe_info;
>> +
>> + return 0;
>> +}
>> +
>> +static void cleanup_worker_threads(struct cpt_vf *cptvf)
>> +{
>> + struct cptvf_wqe_info *cwqe_info;
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int i;
>> +
>> + cwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
>> + if (!cwqe_info)
>> + return;
>> +
>> + if (cptvf->nr_queues) {
>> + dev_info(&pdev->dev, "Cleaning VQ worker threads (%u)\n",
>> + cptvf->nr_queues);
>> + }
>> +
>> + for (i = 0; i < cptvf->nr_queues; i++)
>> + tasklet_kill(&cwqe_info->vq_wqe[i].twork);
>> +
>> + kzfree(cwqe_info);
>> + cptvf->wqe_info = NULL;
>> +}
>> +
>> +static void free_pending_queues(struct pending_qinfo *pqinfo)
>> +{
>> + int i;
>> + struct pending_queue *queue;
>> +
>> + for_each_pending_queue(pqinfo, queue, i) {
>> + if (!queue->head)
>> + continue;
>> +
>> + /* free single queue */
>> + kzfree((queue->head));
>> +
>> + queue->front = 0;
>> + queue->rear = 0;
>> +
>> + return;
>> + }
>> +
>> + pqinfo->qlen = 0;
>> + pqinfo->nr_queues = 0;
>> +}
>> +
>> +static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
>> + u32 nr_queues)
>> +{
>> + u32 i;
>> + size_t size;
>> + int ret;
>> + struct pending_queue *queue = NULL;
>> +
>> + pqinfo->nr_queues = nr_queues;
>> + pqinfo->qlen = qlen;
>> +
>> + size = (qlen * sizeof(struct pending_entry));
>> +
>> + for_each_pending_queue(pqinfo, queue, i) {
>> + queue->head = kzalloc((size), GFP_KERNEL);
>> + if (!queue->head) {
>> + ret = -ENOMEM;
>> + goto pending_qfail;
>> + }
>> +
>> + queue->front = 0;
>> + queue->rear = 0;
>> + atomic64_set((&queue->pending_count), (0));
>> +
>> + /* init queue spin lock */
>> + spin_lock_init(&queue->lock);
>> + }
>> +
>> + return 0;
>> +
>> +pending_qfail:
>> + free_pending_queues(pqinfo);
>> +
>> + return ret;
>> +}
>> +
>> +static int init_pending_queues(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int ret;
>> +
>> + if (!nr_queues)
>> + return 0;
>> +
>> + ret = alloc_pending_queues(&cptvf->pqinfo, qlen, nr_queues);
>> + if (ret) {
>> + dev_err(&pdev->dev, "failed to setup pending queues (%u)\n",
>> + nr_queues);
>> + return ret;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +static void cleanup_pending_queues(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (!cptvf->nr_queues)
>> + return;
>> +
>> + dev_info(&pdev->dev, "Cleaning VQ pending queue (%u)\n",
>> + cptvf->nr_queues);
>> + free_pending_queues(&cptvf->pqinfo);
>> +}
>> +
>> +static void free_command_queues(struct cpt_vf *cptvf,
>> + struct command_qinfo *cqinfo)
>> +{
>> + int i, j;
>> + struct command_queue *queue = NULL;
>> + struct command_chunk *chunk = NULL, *next = NULL;
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct hlist_node *node;
>> +
>> + /* clean up for each queue */
>> + for (i = 0; i < cptvf->nr_queues; i++) {
>> + queue = &cqinfo->queue[i];
>> + if (hlist_empty(&cqinfo->queue[i].chead))
>> + continue;
>> +
>> + hlist_for_each(node, &cqinfo->queue[i].chead) {
>> + chunk = hlist_entry(node, struct command_chunk,
>> + nextchunk);
>> + break;
>> + }
> What exactly is the purpose of that loop?
>
>> + for (j = 0; j < queue->nchunks; j++) {
>> + if (j < queue->nchunks) {
> We already know that "j < queue->nchunks" at this point...
>
>> + node = node->next;
>> + next = hlist_entry(node, struct command_chunk,
>> + nextchunk);
>> + }
>> +
>> + dma_free_coherent(&pdev->dev, chunk->size,
>> + chunk->head,
>> + chunk->dma_addr);
>> + chunk->head = NULL;
>> + chunk->dma_addr = 0;
>> + hlist_del(&chunk->nextchunk);
>> + kzfree(chunk);
>> + chunk = next;
>> + }
>> + queue->nchunks = 0;
>> + queue->idx = 0;
>> + }
> This whole function looks like an attempt to open code
> hlist_for_each_entry_safe(), why didn't you just use that?
Yes you are right I could use hlist_for_each_entry_safe(). will fix it
in next version
>> +
>> + /* common cleanup */
>> + cqinfo->cmd_size = 0;
>> +}
>> +
>> +static int alloc_command_queues(struct cpt_vf *cptvf,
>> + struct command_qinfo *cqinfo, size_t cmd_size,
>> + u32 qlen)
>> +{
>> + int i;
>> + size_t q_size;
>> + struct command_queue *queue = NULL;
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + /* common init */
>> + cqinfo->cmd_size = cmd_size;
>> + /* Qsize in dwords, needed for SADDR config, 1-next chunk pointer */
>> + cptvf->qsize = min(qlen, cqinfo->qchunksize) *
>> + CPT_NEXT_CHUNK_PTR_SIZE + 1;
>> + /* Qsize in bytes to create space for alignment */
>> + q_size = qlen * cqinfo->cmd_size;
>> +
>> + /* per queue initialization */
>> + for (i = 0; i < cptvf->nr_queues; i++) {
>> + size_t c_size = 0;
>> + size_t rem_q_size = q_size;
>> + struct command_chunk *curr = NULL, *first = NULL, *last = NULL;
>> + u32 qcsize_bytes = cqinfo->qchunksize * cqinfo->cmd_size;
>> +
>> + queue = &cqinfo->queue[i];
>> + INIT_HLIST_HEAD(&cqinfo->queue[i].chead);
>> + do {
>> + curr = kzalloc(sizeof(*curr), GFP_KERNEL);
>> + if (!curr)
>> + goto cmd_qfail;
>> +
>> + c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes :
>> + rem_q_size;
>> + curr->head = (u8 *)dma_zalloc_coherent(&pdev->dev,
>> + c_size + CPT_NEXT_CHUNK_PTR_SIZE,
>> + &curr->dma_addr, GFP_KERNEL);
>> + if (!curr->head) {
>> + dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n",
>> + i, queue->nchunks);
>> + goto cmd_qfail;
>> + }
>> +
>> + curr->size = c_size;
>> + if (queue->nchunks == 0) {
>> + hlist_add_head(&curr->nextchunk,
>> + &cqinfo->queue[i].chead);
>> + first = curr;
>> + } else {
>> + hlist_add_behind(&curr->nextchunk,
>> + &last->nextchunk);
>> + }
>> +
>> + queue->nchunks++;
>> + rem_q_size -= c_size;
>> + if (last)
>> + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
>> +
>> + last = curr;
>> + } while (rem_q_size);
>> +
>> + /* Make the queue circular */
>> + /* Tie back last chunk entry to head */
>> + curr = first;
>> + *((u64 *)(&last->head[last->size])) = (u64)curr->dma_addr;
>> + last->nextchunk.next = &curr->nextchunk;
> You shouldn't access the hlist struct members directly, use helper
> functions here.
okay
>> + queue->qhead = curr;
>> + spin_lock_init(&queue->lock);
>> + }
>> + return 0;
>> +
>> +cmd_qfail:
>> + free_command_queues(cptvf, cqinfo);
>> + return -ENOMEM;
>> +}
>> +
>> +static int init_command_queues(struct cpt_vf *cptvf, u32 qlen)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int ret;
>> +
>> + /* setup AE command queues */
>> + ret = alloc_command_queues(cptvf, &cptvf->cqinfo, CPT_INST_SIZE,
>> + qlen);
>> + if (ret) {
>> + dev_err(&pdev->dev, "failed to allocate AE command queues (%u)\n",
>> + cptvf->nr_queues);
>> + return ret;
>> + }
>> +
>> + return ret;
>> +}
>> +
>> +static void cleanup_command_queues(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (!cptvf->nr_queues)
>> + return;
>> +
>> + dev_info(&pdev->dev, "Cleaning VQ command queue (%u)\n",
>> + cptvf->nr_queues);
>> + free_command_queues(cptvf, &cptvf->cqinfo);
>> +}
>> +
>> +static void cptvf_sw_cleanup(struct cpt_vf *cptvf)
>> +{
>> + cleanup_worker_threads(cptvf);
>> + cleanup_pending_queues(cptvf);
>> + cleanup_command_queues(cptvf);
>> +}
>> +
>> +static int cptvf_sw_init(struct cpt_vf *cptvf, u32 qlen, u32 nr_queues)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int ret = 0;
>> + u32 max_dev_queues = 0;
>> +
>> + max_dev_queues = CPT_NUM_QS_PER_VF;
>> + /* possible cpus */
>> + nr_queues = min_t(u32, nr_queues, max_dev_queues);
>> + cptvf->nr_queues = nr_queues;
>> +
>> + ret = init_command_queues(cptvf, qlen);
>> + if (ret) {
>> + dev_err(&pdev->dev, "Failed to setup command queues (%u)\n",
>> + nr_queues);
>> + return ret;
>> + }
>> +
>> + ret = init_pending_queues(cptvf, qlen, nr_queues);
>> + if (ret) {
>> + dev_err(&pdev->dev, "Failed to setup pending queues (%u)\n",
>> + nr_queues);
>> + goto setup_pqfail;
>> + }
>> +
>> + /* Create worker threads for BH processing */
>> + ret = init_worker_threads(cptvf);
>> + if (ret) {
>> + dev_err(&pdev->dev, "Failed to setup worker threads\n");
>> + goto init_work_fail;
>> + }
>> +
>> + return 0;
>> +
>> +init_work_fail:
>> + cleanup_worker_threads(cptvf);
>> + cleanup_pending_queues(cptvf);
>> +
>> +setup_pqfail:
>> + cleanup_command_queues(cptvf);
>> +
>> + return ret;
>> +}
>> +
>> +static void cptvf_disable_msix(struct cpt_vf *cptvf)
>> +{
>> + if (cptvf->msix_enabled) {
>> + pci_disable_msix(cptvf->pdev);
>> + cptvf->msix_enabled = 0;
>> + }
>> +}
>> +
>> +static int cptvf_enable_msix(struct cpt_vf *cptvf)
>> +{
>> + int i, ret;
>> +
>> + for (i = 0; i < CPT_VF_MSIX_VECTORS; i++)
>> + cptvf->msix_entries[i].entry = i;
>> +
>> + ret = pci_enable_msix(cptvf->pdev, cptvf->msix_entries,
>> + CPT_VF_MSIX_VECTORS);
>> + if (ret) {
>> + dev_err(&cptvf->pdev->dev, "Request for #%d msix vectors failed\n",
>> + CPT_VF_MSIX_VECTORS);
>> + return ret;
>> + }
>> +
>> + cptvf->msix_enabled = 1;
>> + /* Mark MSIX enabled */
>> + cptvf->flags |= CPT_FLAG_MSIX_ENABLED;
>> +
>> + return 0;
>> +}
>> +
>> +static void cptvf_free_all_interrupts(struct cpt_vf *cptvf)
>> +{
>> + int irq;
>> +
>> + for (irq = 0; irq < CPT_VF_MSIX_VECTORS; irq++) {
>> + if (cptvf->irq_allocated[irq])
>> + irq_set_affinity_hint(cptvf->msix_entries[irq].vector,
>> + NULL);
>> + free_cpumask_var(cptvf->affinity_mask[irq]);
>> + free_irq(cptvf->msix_entries[irq].vector, cptvf);
>> + cptvf->irq_allocated[irq] = false;
>> + }
>> +}
>> +
>> +static void cptvf_write_vq_ctl(struct cpt_vf *cptvf, bool val)
>> +{
>> + union cptx_vqx_ctl vqx_ctl;
>> +
>> + vqx_ctl.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0));
>> + vqx_ctl.s.ena = val;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_CTL(0, 0), vqx_ctl.u);
>> +}
>> +
>> +void cptvf_write_vq_doorbell(struct cpt_vf *cptvf, u32 val)
>> +{
>> + union cptx_vqx_doorbell vqx_dbell;
>> +
>> + vqx_dbell.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_DOORBELL(0, 0));
>> + vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DOORBELL(0, 0),
>> + vqx_dbell.u);
>> +}
>> +
>> +static void cptvf_write_vq_inprog(struct cpt_vf *cptvf, u8 val)
>> +{
>> + union cptx_vqx_inprog vqx_inprg;
>> +
>> + vqx_inprg.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0));
>> + vqx_inprg.s.inflight = val;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
>> +}
>> +
>> +static void cptvf_write_vq_done_numwait(struct cpt_vf *cptvf, u32 val)
>> +{
>> + union cptx_vqx_done_wait vqx_dwait;
>> +
>> + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_DONE_WAIT(0, 0));
>> + vqx_dwait.s.num_wait = val;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
>> + vqx_dwait.u);
>> +}
>> +
>> +static void cptvf_write_vq_done_timewait(struct cpt_vf *cptvf, u16 time)
>> +{
>> + union cptx_vqx_done_wait vqx_dwait;
>> +
>> + vqx_dwait.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_DONE_WAIT(0, 0));
>> + vqx_dwait.s.time_wait = time;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_WAIT(0, 0),
>> + vqx_dwait.u);
>> +}
>> +
>> +static void cptvf_enable_swerr_interrupts(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_ena_w1s vqx_misc_ena;
>> +
>> + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_ENA_W1S(0, 0));
>> + /* Set mbox(0) interupts for the requested vf */
>> + vqx_misc_ena.s.swerr = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
>> + vqx_misc_ena.u);
>> +}
>> +
>> +static void cptvf_enable_mbox_interrupts(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_ena_w1s vqx_misc_ena;
>> +
>> + vqx_misc_ena.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_ENA_W1S(0, 0));
>> + /* Set mbox(0) interupts for the requested vf */
>> + vqx_misc_ena.s.mbox = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_ENA_W1S(0, 0),
>> + vqx_misc_ena.u);
>> +}
>> +
>> +static void cptvf_enable_done_interrupts(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_done_ena_w1s vqx_done_ena;
>> +
>> + vqx_done_ena.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_DONE_ENA_W1S(0, 0));
>> + /* Set DONE interrupt for the requested vf */
>> + vqx_done_ena.s.done = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ENA_W1S(0, 0),
>> + vqx_done_ena.u);
>> +}
>> +
>> +static void cptvf_clear_dovf_intr(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_int vqx_misc_int;
>> +
>> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_INT(0, 0));
>> + /* W1C for the VF */
>> + vqx_misc_int.s.dovf = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
>> + vqx_misc_int.u);
>> +}
>> +
>> +static void cptvf_clear_irde_intr(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_int vqx_misc_int;
>> +
>> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_INT(0, 0));
>> + /* W1C for the VF */
>> + vqx_misc_int.s.irde = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
>> + vqx_misc_int.u);
>> +}
>> +
>> +static void cptvf_clear_nwrp_intr(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_int vqx_misc_int;
>> +
>> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_INT(0, 0));
>> + /* W1C for the VF */
>> + vqx_misc_int.s.nwrp = 1;
>> + cpt_write_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
>> +}
>> +
>> +static void cptvf_clear_mbox_intr(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_int vqx_misc_int;
>> +
>> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_INT(0, 0));
>> + /* W1C for the VF */
>> + vqx_misc_int.s.mbox = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
>> + vqx_misc_int.u);
>> +}
>> +
>> +static void cptvf_clear_swerr_intr(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_misc_int vqx_misc_int;
>> +
>> + vqx_misc_int.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_MISC_INT(0, 0));
>> + /* W1C for the VF */
>> + vqx_misc_int.s.swerr = 1;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0),
>> + vqx_misc_int.u);
>> +}
>> +
>> +static u64 cptvf_read_vf_misc_intr_status(struct cpt_vf *cptvf)
>> +{
>> + return cpt_read_csr64(cptvf->reg_base, CPTX_VQX_MISC_INT(0, 0));
>> +}
>> +
>> +static irqreturn_t cptvf_misc_intr_handler(int irq, void *cptvf_irq)
>> +{
>> + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
>> + struct pci_dev *pdev = cptvf->pdev;
>> + u64 intr;
>> +
>> + intr = cptvf_read_vf_misc_intr_status(cptvf);
>> + /*Check for MISC interrupt types*/
>> + if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
>> + dev_err(&pdev->dev, "Mailbox interrupt 0x%llx on CPT VF %d\n",
>> + intr, cptvf->vfid);
>> + cptvf_handle_mbox_intr(cptvf);
>> + cptvf_clear_mbox_intr(cptvf);
>> + } else if (unlikely(intr & CPT_VF_INTR_DOVF_MASK)) {
>> + cptvf_clear_dovf_intr(cptvf);
>> + /*Clear doorbell count*/
>> + cptvf_write_vq_doorbell(cptvf, 0);
>> + dev_err(&pdev->dev, "Doorbell overflow error interrupt 0x%llx on CPT VF %d\n",
>> + intr, cptvf->vfid);
>> + } else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
>> + cptvf_clear_irde_intr(cptvf);
>> + dev_err(&pdev->dev, "Instruction NCB read error interrupt 0x%llx on CPT VF %d\n",
>> + intr, cptvf->vfid);
>> + } else if (unlikely(intr & CPT_VF_INTR_NWRP_MASK)) {
>> + cptvf_clear_nwrp_intr(cptvf);
>> + dev_err(&pdev->dev, "NCB response write error interrupt 0x%llx on CPT VF %d\n",
>> + intr, cptvf->vfid);
>> + } else if (unlikely(intr & CPT_VF_INTR_SERR_MASK)) {
>> + cptvf_clear_swerr_intr(cptvf);
>> + dev_err(&pdev->dev, "Software error interrupt 0x%llx on CPT VF %d\n",
>> + intr, cptvf->vfid);
>> + } else {
>> + dev_err(&pdev->dev, "Unhandled interrupt in CPT VF %d\n",
>> + cptvf->vfid);
>> + }
>> +
>> + return IRQ_HANDLED;
>> +}
>> +
>> +static inline struct cptvf_wqe *get_cptvf_vq_wqe(struct cpt_vf *cptvf,
>> + int qno)
>> +{
>> + struct cptvf_wqe_info *nwqe_info;
>> +
>> + if (unlikely(qno >= cptvf->nr_queues))
>> + return NULL;
>> + nwqe_info = (struct cptvf_wqe_info *)cptvf->wqe_info;
>> +
>> + return &nwqe_info->vq_wqe[qno];
>> +}
>> +
>> +static inline u32 cptvf_read_vq_done_count(struct cpt_vf *cptvf)
>> +{
>> + union cptx_vqx_done vqx_done;
>> +
>> + vqx_done.u = cpt_read_csr64(cptvf->reg_base, CPTX_VQX_DONE(0, 0));
>> + return vqx_done.s.done;
>> +}
>> +
>> +static inline void cptvf_write_vq_done_ack(struct cpt_vf *cptvf,
>> + u32 ackcnt)
>> +{
>> + union cptx_vqx_done_ack vqx_dack_cnt;
>> +
>> + vqx_dack_cnt.u = cpt_read_csr64(cptvf->reg_base,
>> + CPTX_VQX_DONE_ACK(0, 0));
>> + vqx_dack_cnt.s.done_ack = ackcnt;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_DONE_ACK(0, 0),
>> + vqx_dack_cnt.u);
>> +}
>> +
>> +static irqreturn_t cptvf_done_intr_handler(int irq, void *cptvf_irq)
>> +{
>> + struct cpt_vf *cptvf = (struct cpt_vf *)cptvf_irq;
>> + struct pci_dev *pdev = cptvf->pdev;
>> + /* Read the number of completions */
>> + u32 intr = cptvf_read_vq_done_count(cptvf);
>> +
>> + if (intr) {
>> + struct cptvf_wqe *wqe;
>> +
>> + /* Acknowledge the number of
>> + * scheduled completions for processing
>> + */
>> + cptvf_write_vq_done_ack(cptvf, intr);
>> + wqe = get_cptvf_vq_wqe(cptvf, 0);
>> + if (unlikely(!wqe)) {
>> + dev_err(&pdev->dev, "No work to schedule for VF (%d)",
>> + cptvf->vfid);
>> + return IRQ_NONE;
>> + }
>> + tasklet_hi_schedule(&wqe->twork);
>> + }
>> +
>> + return IRQ_HANDLED;
>> +}
>> +
>> +static int cptvf_register_misc_intr(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int ret;
>> +
>> + /* Register misc interrupt handlers */
>> + ret = request_irq(cptvf->msix_entries[CPT_VF_INT_VEC_E_MISC].vector,
>> + cptvf_misc_intr_handler, 0, "CPT VF misc intr",
>> + cptvf);
>> + if (ret)
>> + goto fail;
>> +
>> + cptvf->irq_allocated[CPT_VF_INT_VEC_E_MISC] = true;
>> +
>> + /* Enable mailbox interrupt */
>> + cptvf_enable_mbox_interrupts(cptvf);
>> + cptvf_enable_swerr_interrupts(cptvf);
>> +
>> + return 0;
>> +
>> +fail:
>> + dev_err(&pdev->dev, "Request misc irq failed");
>> + cptvf_free_all_interrupts(cptvf);
>> + return ret;
>> +}
>> +
>> +static int cptvf_register_done_intr(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int ret;
>> +
>> + /* Register DONE interrupt handlers */
>> + ret = request_irq(cptvf->msix_entries[CPT_VF_INT_VEC_E_DONE].vector,
>> + cptvf_done_intr_handler, 0, "CPT VF done intr",
>> + cptvf);
>> + if (ret)
>> + goto fail;
>> +
>> + cptvf->irq_allocated[CPT_VF_INT_VEC_E_DONE] = true;
>> +
>> + /* Enable mailbox interrupt */
>> + cptvf_enable_done_interrupts(cptvf);
>> + return 0;
>> +
>> +fail:
>> + dev_err(&pdev->dev, "Request done irq failed\n");
>> + cptvf_free_all_interrupts(cptvf);
>> + return ret;
>> +}
>> +
>> +static void cptvf_unregister_interrupts(struct cpt_vf *cptvf)
>> +{
>> + cptvf_free_all_interrupts(cptvf);
>> + cptvf_disable_msix(cptvf);
>> +}
>> +
>> +static void cptvf_set_irq_affinity(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + int vec, cpu;
>> + int irqnum;
>> +
>> + for (vec = 0; vec < CPT_VF_MSIX_VECTORS; vec++) {
>> + if (!cptvf->irq_allocated[vec])
>> + continue;
>> +
>> + if (!zalloc_cpumask_var(&cptvf->affinity_mask[vec],
>> + GFP_KERNEL)) {
>> + dev_err(&pdev->dev, "Allocation failed for affinity_mask for VF %d",
>> + cptvf->vfid);
>> + return;
>> + }
>> +
>> + cpu = cptvf->vfid % num_online_cpus();
>> + cpumask_set_cpu(cpumask_local_spread(cpu, cptvf->node),
>> + cptvf->affinity_mask[vec]);
>> + irqnum = cptvf->msix_entries[vec].vector;
>> + irq_set_affinity_hint(irqnum, cptvf->affinity_mask[vec]);
>> + }
>> +}
>> +
>> +static void cptvf_write_vq_saddr(struct cpt_vf *cptvf, u64 val)
>> +{
>> + union cptx_vqx_saddr vqx_saddr;
>> +
>> + vqx_saddr.u = val;
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
>> +}
>> +
>> +void cptvf_device_init(struct cpt_vf *cptvf)
>> +{
>> + u64 base_addr = 0;
>> +
>> + /* Disable the VQ */
>> + cptvf_write_vq_ctl(cptvf, 0);
>> + /* Reset the doorbell */
>> + cptvf_write_vq_doorbell(cptvf, 0);
>> + /* Clear inflight */
>> + cptvf_write_vq_inprog(cptvf, 0);
>> + /* Write VQ SADDR */
>> + /* TODO: for now only one queue, so hard coded */
>> + base_addr = (u64)(cptvf->cqinfo.queue[0].qhead->dma_addr);
>> + cptvf_write_vq_saddr(cptvf, base_addr);
>> + /* Configure timerhold / coalescence */
>> + cptvf_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
>> + cptvf_write_vq_done_numwait(cptvf, 1);
>> + /* Enable the VQ */
>> + cptvf_write_vq_ctl(cptvf, 1);
>> + /* Flag the VF ready */
>> + cptvf->flags |= CPT_FLAG_DEVICE_READY;
>> +}
>> +
>> +static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>> +{
>> + struct device *dev = &pdev->dev;
>> + struct cpt_vf *cptvf;
>> + int err;
>> +
>> + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL);
>> + if (!cptvf)
>> + return -ENOMEM;
>> +
>> + pci_set_drvdata(pdev, cptvf);
>> + cptvf->pdev = pdev;
>> + err = pci_enable_device(pdev);
>> + if (err) {
>> + dev_err(dev, "Failed to enable PCI device\n");
>> + pci_set_drvdata(pdev, NULL);
>> + return err;
>> + }
>> +
>> + err = pci_request_regions(pdev, DRV_NAME);
>> + if (err) {
>> + dev_err(dev, "PCI request regions failed 0x%x\n", err);
>> + goto cptvf_err_disable_device;
>> + }
>> + /* Mark as VF driver */
>> + cptvf->flags |= CPT_FLAG_VF_DRIVER;
>> + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
>> + if (err) {
>> + dev_err(dev, "Unable to get usable DMA configuration\n");
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
>> + if (err) {
>> + dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n");
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + /* MAP PF's configuration registers */
>> + cptvf->reg_base = pcim_iomap(pdev, 0, 0);
>> + if (!cptvf->reg_base) {
>> + dev_err(dev, "Cannot map config register space, aborting\n");
>> + err = -ENOMEM;
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + cptvf->node = dev_to_node(&pdev->dev);
>> + /* Enable MSI-X */
>> + err = cptvf_enable_msix(cptvf);
>> + if (err) {
>> + dev_err(dev, "cptvf_enable_msix() failed");
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + /* Register mailbox interrupts */
>> + cptvf_register_misc_intr(cptvf);
>> +
>> + /* Check ready with PF */
>> + /* Gets chip ID / device Id from PF if ready */
>> + err = cptvf_check_pf_ready(cptvf);
>> + if (err) {
>> + dev_err(dev, "PF not responding to READY msg");
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + /* CPT VF software resources initialization */
>> + cptvf->cqinfo.qchunksize = CPT_CMD_QCHUNK_SIZE;
>> + err = cptvf_sw_init(cptvf, CPT_CMD_QLEN, CPT_NUM_QS_PER_VF);
>> + if (err) {
>> + dev_err(dev, "cptvf_sw_init() failed");
>> + goto cptvf_err_release_regions;
>> + }
>> + /* Convey VQ LEN to PF */
>> + err = cptvf_send_vq_size_msg(cptvf);
>> + if (err) {
>> + dev_err(dev, "PF not responding to QLEN msg");
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + /* CPT VF device initialization */
>> + cptvf_device_init(cptvf);
>> + /* Send msg to PF to assign currnet Q to required group */
>> + cptvf->vfgrp = 1;
>> + err = cptvf_send_vf_to_grp_msg(cptvf);
>> + if (err) {
>> + dev_err(dev, "PF not responding to VF_GRP msg");
>> + goto cptvf_err_release_regions;
>> + }
>> +
>> + cptvf->priority = 1;
>> + err = cptvf_send_vf_priority_msg(cptvf);
>> + if (err) {
>> + dev_err(dev, "PF not responding to VF_PRIO msg");
>> + goto cptvf_err_release_regions;
>> + }
>> + /* Register DONE interrupts */
>> + err = cptvf_register_done_intr(cptvf);
>> + if (err)
>> + goto cptvf_err_release_regions;
>> +
>> + /* Set irq affinity masks */
>> + cptvf_set_irq_affinity(cptvf);
>> + /* Convey UP to PF */
>> + err = cptvf_send_vf_up(cptvf);
>> + if (err) {
>> + dev_err(dev, "PF not responding to UP msg");
>> + goto cptvf_up_fail;
>> + }
>> + err = cvm_crypto_init(cptvf);
>> + if (err) {
>> + dev_err(dev, "Algorithm register failed\n");
>> + goto cptvf_up_fail;
>> + }
>> + return 0;
>> +
>> +cptvf_up_fail:
>> + cptvf_unregister_interrupts(cptvf);
>> +cptvf_err_release_regions:
>> + pci_release_regions(pdev);
>> +cptvf_err_disable_device:
>> + pci_disable_device(pdev);
>> + pci_set_drvdata(pdev, NULL);
>> +
>> + return err;
>> +}
>> +
>> +static void cptvf_remove(struct pci_dev *pdev)
>> +{
>> + struct cpt_vf *cptvf = pci_get_drvdata(pdev);
>> +
>> + if (!cptvf)
>> + dev_err(&pdev->dev, "Invalid CPT-VF device\n");
>> +
>> + /* Convey DOWN to PF */
>> + if (cptvf_send_vf_down(cptvf)) {
>> + dev_err(&pdev->dev, "PF not responding to DOWN msg");
>> + } else {
>> + cptvf_unregister_interrupts(cptvf);
>> + cptvf_sw_cleanup(cptvf);
>> + pci_set_drvdata(pdev, NULL);
>> + pci_release_regions(pdev);
>> + pci_disable_device(pdev);
>> + cvm_crypto_exit();
>> + }
>> +}
>> +
>> +static void cptvf_shutdown(struct pci_dev *pdev)
>> +{
>> + cptvf_remove(pdev);
>> +}
>> +
>> +/* Supported devices */
>> +static const struct pci_device_id cptvf_id_table[] = {
>> + {PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
>> + { 0, } /* end of table */
>> +};
>> +
>> +static struct pci_driver cptvf_pci_driver = {
>> + .name = DRV_NAME,
>> + .id_table = cptvf_id_table,
>> + .probe = cptvf_probe,
>> + .remove = cptvf_remove,
>> + .shutdown = cptvf_shutdown,
>> +};
>> +
>> +module_pci_driver(cptvf_pci_driver);
>> +
>> +MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
>> +MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
>> +MODULE_LICENSE("GPL v2");
>> +MODULE_VERSION(DRV_VERSION);
>> +MODULE_DEVICE_TABLE(pci, cptvf_id_table);
>> diff --git a/drivers/crypto/cavium/cpt/cptvf_mbox.c b/drivers/crypto/cavium/cpt/cptvf_mbox.c
>> new file mode 100644
>> index 0000000..d5ec3b8
>> --- /dev/null
>> +++ b/drivers/crypto/cavium/cpt/cptvf_mbox.c
>> @@ -0,0 +1,211 @@
>> +/*
>> + * Copyright (C) 2016 Cavium, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of version 2 of the GNU General Public License
>> + * as published by the Free Software Foundation.
>> + */
>> +
>> +#include "cptvf.h"
>> +
>> +static void cptvf_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
>> +{
>> + /* Writing mbox(1) causes interrupt */
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0),
>> + mbx->msg);
>> + cpt_write_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1),
>> + mbx->data);
>> +}
>> +
>> +/* ACKs PF's mailbox message
>> + */
>> +void cptvf_mbox_send_ack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
>> +{
>> + mbx->msg = CPT_MBOX_MSG_TYPE_ACK;
>> + cptvf_send_msg_to_pf(cptvf, mbx);
>> +}
>> +
>> +/* NACKs PF's mailbox message that VF is not able to
>> + * complete the action
>> + */
>> +void cptvf_mbox_send_nack(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
>> +{
>> + mbx->msg = CPT_MBOX_MSG_TYPE_NACK;
>> + cptvf_send_msg_to_pf(cptvf, mbx);
>> +}
>> +
>> +/* Interrupt handler to handle mailbox messages from VFs */
>> +void cptvf_handle_mbox_intr(struct cpt_vf *cptvf)
>> +{
>> + struct cpt_mbox mbx = {};
>> +
>> + /*
>> + * MBOX[0] contains msg
>> + * MBOX[1] contains data
>> + */
>> + mbx.msg = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 0));
>> + mbx.data = cpt_read_csr64(cptvf->reg_base, CPTX_VFX_PF_MBOXX(0, 0, 1));
>> + dev_dbg(&cptvf->pdev->dev, "%s: Mailbox msg 0x%llx from PF\n",
>> + __func__, mbx.msg);
>> + switch (mbx.msg) {
>> + case CPT_MSG_READY:
>> + {
>> + cptvf->pf_acked = true;
>> + cptvf->vfid = mbx.data;
>> + dev_dbg(&cptvf->pdev->dev, "Received VFID %d\n", cptvf->vfid);
>> + break;
>> + }
>> + case CPT_MSG_QBIND_GRP:
>> + cptvf->pf_acked = true;
>> + cptvf->vftype = mbx.data;
>> + dev_dbg(&cptvf->pdev->dev, "VF %d type %s group %d\n",
>> + cptvf->vfid, ((mbx.data == SE_TYPES) ? "SE" : "AE"),
>> + cptvf->vfgrp);
>> + break;
>> + case CPT_MBOX_MSG_TYPE_ACK:
>> + cptvf->pf_acked = true;
>> + break;
>> + case CPT_MBOX_MSG_TYPE_NACK:
>> + cptvf->pf_nacked = true;
>> + break;
>> + default:
>> + dev_err(&cptvf->pdev->dev, "Invalid msg from PF, msg 0x%llx\n",
>> + mbx.msg);
>> + break;
>> + }
>> +}
>> +
>> +static int cptvf_send_msg_to_pf_timeout(struct cpt_vf *cptvf,
>> + struct cpt_mbox *mbx)
>> +{
>> + int timeout = CPT_MBOX_MSG_TIMEOUT;
>> + int sleep = 10;
>> +
>> + cptvf->pf_acked = false;
>> + cptvf->pf_nacked = false;
>> + cptvf_send_msg_to_pf(cptvf, mbx);
>> + /* Wait for previous message to be acked, timeout 2sec */
>> + while (!cptvf->pf_acked) {
>> + if (cptvf->pf_nacked)
>> + return -EINVAL;
>> + msleep(sleep);
>> + if (cptvf->pf_acked)
>> + break;
>> + timeout -= sleep;
>> + if (!timeout) {
>> + dev_err(&cptvf->pdev->dev, "PF didn't ack to mbox msg %llx from VF%u\n",
>> + (mbx->msg & 0xFF), cptvf->vfid);
>> + return -EBUSY;
>> + }
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +/*
>> + * Checks if VF is able to comminicate with PF
>> + * and also gets the CPT number this VF is associated to.
>> + */
>> +int cptvf_check_pf_ready(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_mbox mbx = {};
>> +
>> + mbx.msg = CPT_MSG_READY;
>> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
>> + dev_err(&pdev->dev, "PF didn't respond to READY msg\n");
>> + return -EBUSY;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +/*
>> + * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
>> + * Must be ACKed.
>> + */
>> +int cptvf_send_vq_size_msg(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_mbox mbx = {};
>> +
>> + mbx.msg = CPT_MSG_QLEN;
>> + mbx.data = cptvf->qsize;
>> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
>> + dev_err(&pdev->dev, "PF didn't respond to vq_size msg\n");
>> + return -EBUSY;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +/*
>> + * Communicate VF group required to PF and get the VQ binded to that group
>> + */
>> +int cptvf_send_vf_to_grp_msg(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_mbox mbx = {};
>> +
>> + mbx.msg = CPT_MSG_QBIND_GRP;
>> + /* Convey group of the VF */
>> + mbx.data = cptvf->vfgrp;
>> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
>> + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
>> + return -EBUSY;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +/*
>> + * Communicate VF group required to PF and get the VQ binded to that group
>> + */
>> +int cptvf_send_vf_priority_msg(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_mbox mbx = {};
>> +
>> + mbx.msg = CPT_MSG_VQ_PRIORITY;
>> + /* Convey group of the VF */
>> + mbx.data = cptvf->priority;
>> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
>> + dev_err(&pdev->dev, "PF didn't respond to vf_type msg\n");
>> + return -EBUSY;
>> + }
>> + return 0;
>> +}
>> +
>> +/*
>> + * Communicate to PF that VF is UP and running
>> + */
>> +int cptvf_send_vf_up(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_mbox mbx = {};
>> +
>> + mbx.msg = CPT_MSG_VF_UP;
>> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
>> + dev_err(&pdev->dev, "PF didn't respond to UP msg\n");
>> + return -EBUSY;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +/*
>> + * Communicate to PF that VF is DOWN and running
>> + */
>> +int cptvf_send_vf_down(struct cpt_vf *cptvf)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_mbox mbx = {};
>> +
>> + mbx.msg = CPT_MSG_VF_DOWN;
>> + if (cptvf_send_msg_to_pf_timeout(cptvf, &mbx)) {
>> + dev_err(&pdev->dev, "PF didn't respond to DOWN msg\n");
>> + return -EBUSY;
>> + }
>> +
>> + return 0;
>> +}
>> diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
>> new file mode 100644
>> index 0000000..062b8e9
>> --- /dev/null
>> +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
>> @@ -0,0 +1,593 @@
>> +/*
>> + * Copyright (C) 2016 Cavium, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of version 2 of the GNU General Public License
>> + * as published by the Free Software Foundation.
>> + */
>> +
>> +#include "cptvf.h"
>> +#include "request_manager.h"
>> +
>> +/**
>> + * get_free_pending_entry - get free entry from pending queue
>> + * @param pqinfo: pending_qinfo structure
>> + * @param qno: queue number
>> + */
>> +static struct pending_entry *get_free_pending_entry(struct pending_queue *q,
>> + int qlen)
>> +{
>> + struct pending_entry *ent = NULL;
>> +
>> + ent = &q->head[q->rear];
>> + if (unlikely(ent->busy)) {
>> + ent = NULL;
>> + goto no_free_entry;
>> + }
>> +
>> + q->rear++;
>> + if (unlikely(q->rear == qlen))
>> + q->rear = 0;
>> +
>> +no_free_entry:
>> + return ent;
>> +}
>> +
>> +static inline void pending_queue_inc_front(struct pending_qinfo *pqinfo,
>> + int qno)
>> +{
>> + struct pending_queue *queue = &pqinfo->queue[qno];
>> +
>> + queue->front++;
>> + if (unlikely(queue->front == pqinfo->qlen))
>> + queue->front = 0;
>> +}
>> +
>> +static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
>> + int buf_count, u8 *buffer)
>> +{
>> + int ret = 0, i, j;
>> + int components;
>> + struct sglist_component *sg_ptr = NULL;
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (unlikely(!list)) {
>> + dev_err(&pdev->dev, "Input List pointer is NULL\n");
>> + return -EFAULT;
>> + }
>> +
>> + for (i = 0; i < buf_count; i++) {
>> + if (likely(list[i].vptr)) {
>> + list[i].dma_addr = dma_map_single(&pdev->dev,
>> + list[i].vptr,
>> + list[i].size,
>> + DMA_BIDIRECTIONAL);
>> + if (unlikely(dma_mapping_error(&pdev->dev,
>> + list[i].dma_addr))) {
>> + dev_err(&pdev->dev, "DMA map kernel buffer failed for component: %d\n",
>> + i);
>> + ret = -EIO;
>> + goto sg_cleanup;
>> + }
>> + }
>> + }
>> +
>> + components = buf_count / 4;
>> + sg_ptr = (struct sglist_component *)buffer;
>> + for (i = 0; i < components; i++) {
>> + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
>> + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
>> + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
>> + sg_ptr->u.s.len3 = cpu_to_be16(list[i * 4 + 3].size);
>> + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
>> + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
>> + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
>> + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr);
>> + sg_ptr++;
>> + }
>> +
>> + components = buf_count % 4;
>> +
>> + switch (components) {
>> + case 3:
>> + sg_ptr->u.s.len2 = cpu_to_be16(list[i * 4 + 2].size);
>> + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr);
>> + /* Fall through */
>> + case 2:
>> + sg_ptr->u.s.len1 = cpu_to_be16(list[i * 4 + 1].size);
>> + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr);
>> + /* Fall through */
>> + case 1:
>> + sg_ptr->u.s.len0 = cpu_to_be16(list[i * 4 + 0].size);
>> + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr);
>> + break;
>> + default:
>> + break;
>> + }
>> +
>> + return ret;
>> +
>> +sg_cleanup:
>> + for (j = 0; j < i; j++) {
>> + if (list[j].dma_addr) {
>> + dma_unmap_single(&pdev->dev, list[i].dma_addr,
>> + list[i].size, DMA_BIDIRECTIONAL);
>> + }
>> +
>> + list[j].dma_addr = 0;
>> + }
>> +
>> + return ret;
>> +}
>> +
>> +static inline int setup_sgio_list(struct cpt_vf *cptvf,
>> + struct cpt_info_buffer *info,
>> + struct cpt_request_info *req)
>> +{
>> + u16 g_sz_bytes = 0, s_sz_bytes = 0;
>> + int ret = 0;
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
>> + dev_err(&pdev->dev, "Request SG components are higher than supported\n");
>> + ret = -EINVAL;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + /* Setup gather (input) components */
>> + g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
>> + info->gather_components = kzalloc(g_sz_bytes, GFP_KERNEL);
>> + if (!info->gather_components) {
>> + ret = -ENOMEM;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + ret = setup_sgio_components(cptvf, req->in,
>> + req->incnt,
>> + info->gather_components);
>> + if (ret) {
>> + dev_err(&pdev->dev, "Failed to setup gather list\n");
>> + ret = -EFAULT;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + /* Setup scatter (output) components */
>> + s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
>> + info->scatter_components = kzalloc(s_sz_bytes, GFP_KERNEL);
>> + if (!info->scatter_components) {
>> + ret = -ENOMEM;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + ret = setup_sgio_components(cptvf, req->out,
>> + req->outcnt,
>> + info->scatter_components);
>> + if (ret) {
>> + dev_err(&pdev->dev, "Failed to setup gather list\n");
>> + ret = -EFAULT;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + /* Create and initialize DPTR */
>> + info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
>> + info->in_buffer = kzalloc(info->dlen, GFP_KERNEL);
>> + if (!info->in_buffer) {
>> + ret = -ENOMEM;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + ((u16 *)info->in_buffer)[0] = req->outcnt;
>> + ((u16 *)info->in_buffer)[1] = req->incnt;
>> + ((u16 *)info->in_buffer)[2] = 0;
>> + ((u16 *)info->in_buffer)[3] = 0;
>> + *(u64 *)info->in_buffer = cpu_to_be64p((u64 *)info->in_buffer);
>> +
>> + memcpy(&info->in_buffer[8], info->gather_components,
>> + g_sz_bytes);
>> + memcpy(&info->in_buffer[8 + g_sz_bytes],
>> + info->scatter_components, s_sz_bytes);
>> +
>> + info->dptr_baddr = dma_map_single(&pdev->dev,
>> + (void *)info->in_buffer,
>> + info->dlen,
>> + DMA_BIDIRECTIONAL);
>> + if (dma_mapping_error(&pdev->dev, info->dptr_baddr)) {
>> + dev_err(&pdev->dev, "Mapping DPTR Failed %d\n", info->dlen);
>> + ret = -EIO;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + /* Create and initialize RPTR */
>> + info->out_buffer = kzalloc(COMPLETION_CODE_SIZE, GFP_KERNEL);
>> + if (!info->out_buffer) {
>> + ret = -ENOMEM;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + *((u64 *)info->out_buffer) = ~((u64)COMPLETION_CODE_INIT);
>> + info->alternate_caddr = (u64 *)info->out_buffer;
>> + info->rptr_baddr = dma_map_single(&pdev->dev,
>> + (void *)info->out_buffer,
>> + COMPLETION_CODE_SIZE,
>> + DMA_BIDIRECTIONAL);
>> + if (dma_mapping_error(&pdev->dev, info->rptr_baddr)) {
>> + dev_err(&pdev->dev, "Mapping RPTR Failed %d\n",
>> + COMPLETION_CODE_SIZE);
>> + ret = -EIO;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + return 0;
>> +
>> +scatter_gather_clean:
>> + return ret;
>> +}
>> +
>> +int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd,
>> + u32 qno)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct command_qinfo *qinfo = NULL;
>> + struct command_queue *queue;
>> + struct command_chunk *chunk;
>> + u8 *ent;
>> + int ret = 0;
>> +
>> + if (unlikely(qno >= cptvf->nr_queues)) {
>> + dev_err(&pdev->dev, "Invalid queue (qno: %d, nr_queues: %d)\n",
>> + qno, cptvf->nr_queues);
>> + return -EINVAL;
>> + }
>> +
>> + qinfo = &cptvf->cqinfo;
>> + queue = &qinfo->queue[qno];
>> + /* lock commad queue */
>> + spin_lock(&queue->lock);
>> + ent = &queue->qhead->head[queue->idx * qinfo->cmd_size];
>> + memcpy(ent, (void *)cmd, qinfo->cmd_size);
>> +
>> + if (++queue->idx >= queue->qhead->size / 64) {
>> + struct hlist_node *node;
>> +
>> + hlist_for_each(node, &queue->chead) {
>> + chunk = hlist_entry(node, struct command_chunk,
>> + nextchunk);
>> + if (chunk == queue->qhead) {
>> + continue;
>> + } else {
>> + queue->qhead = chunk;
>> + break;
>> + }
>> + }
>> + queue->idx = 0;
>> + }
>> + /* make sure all memory stores are done before ringing doorbell */
>> + smp_wmb();
>> + cptvf_write_vq_doorbell(cptvf, 1);
>> + /* unlock command queue */
>> + spin_unlock(&queue->lock);
>> +
>> + return ret;
>> +}
>> +
>> +void do_request_cleanup(struct cpt_vf *cptvf,
>> + struct cpt_info_buffer *info)
>> +{
>> + int i;
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct cpt_request_info *req;
>> +
>> + if (info->dptr_baddr)
>> + dma_unmap_single(&pdev->dev, info->dptr_baddr,
>> + info->dlen, DMA_BIDIRECTIONAL);
>> +
>> + if (info->rptr_baddr)
>> + dma_unmap_single(&pdev->dev, info->rptr_baddr,
>> + COMPLETION_CODE_SIZE, DMA_BIDIRECTIONAL);
>> +
>> + if (info->comp_baddr)
>> + dma_unmap_single(&pdev->dev, info->comp_baddr,
>> + sizeof(union cpt_res_s), DMA_BIDIRECTIONAL);
>> +
>> + if (info->req) {
>> + req = info->req;
>> + for (i = 0; i < req->outcnt; i++) {
>> + if (req->out[i].dma_addr)
>> + dma_unmap_single(&pdev->dev,
>> + req->out[i].dma_addr,
>> + req->out[i].size,
>> + DMA_BIDIRECTIONAL);
>> + }
>> +
>> + for (i = 0; i < req->incnt; i++) {
>> + if (req->in[i].dma_addr)
>> + dma_unmap_single(&pdev->dev,
>> + req->in[i].dma_addr,
>> + req->in[i].size,
>> + DMA_BIDIRECTIONAL);
>> + }
>> + }
>> +
>> + if (info->scatter_components)
>> + kzfree(info->scatter_components);
>> +
>> + if (info->gather_components)
>> + kzfree(info->gather_components);
>> +
>> + if (info->out_buffer)
>> + kzfree(info->out_buffer);
>> +
>> + if (info->in_buffer)
>> + kzfree(info->in_buffer);
>> +
>> + if (info->completion_addr)
>> + kzfree((void *)info->completion_addr);
>> +
>> + kzfree(info);
>> +}
>> +
>> +void do_post_process(struct cpt_vf *cptvf, struct cpt_info_buffer *info)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (!info || !cptvf) {
>> + dev_err(&pdev->dev, "Input params are incorrect for post processing\n");
>> + return;
>> + }
>> +
>> + do_request_cleanup(cptvf, info);
>> +}
>> +
>> +static inline void process_pending_queue(struct cpt_vf *cptvf,
>> + struct pending_qinfo *pqinfo,
>> + int qno)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> + struct pending_queue *pqueue = &pqinfo->queue[qno];
>> + struct pending_entry *pentry = NULL;
>> + struct cpt_info_buffer *info = NULL;
>> + union cpt_res_s *status = NULL;
>> + unsigned char ccode;
>> +
>> + while (1) {
>> + spin_lock_bh(&pqueue->lock);
>> + pentry = &pqueue->head[pqueue->front];
>> + if (unlikely(!pentry->busy)) {
>> + spin_unlock_bh(&pqueue->lock);
>> + break;
>> + }
>> +
>> + info = (struct cpt_info_buffer *)pentry->post_arg;
>> + if (unlikely(!info)) {
>> + dev_err(&pdev->dev, "Pending Entry post arg NULL\n");
>> + pending_queue_inc_front(pqinfo, qno);
>> + spin_unlock_bh(&pqueue->lock);
>> + continue;
>> + }
>> +
>> + status = (union cpt_res_s *)pentry->completion_addr;
>> + ccode = status->s.compcode;
>> + if ((status->s.compcode == CPT_COMP_E_FAULT) ||
>> + (status->s.compcode == CPT_COMP_E_SWERR)) {
>> + dev_err(&pdev->dev, "Request failed with %s\n",
>> + (status->s.compcode == CPT_COMP_E_FAULT) ?
>> + "DMA Fault" : "Software error");
>> + pentry->completion_addr = NULL;
>> + pentry->busy = false;
>> + atomic64_dec((&pqueue->pending_count));
>> + pentry->post_arg = NULL;
>> + pending_queue_inc_front(pqinfo, qno);
>> + do_request_cleanup(cptvf, info);
>> + spin_unlock_bh(&pqueue->lock);
>> + break;
>> + } else if (status->s.compcode == COMPLETION_CODE_INIT) {
>> + /* check for timeout */
>> + if (time_after_eq(jiffies,
>> + (info->time_in +
>> + (CPT_COMMAND_TIMEOUT * HZ)))) {
>> + dev_err(&pdev->dev, "Request timed out");
>> + pentry->completion_addr = NULL;
>> + pentry->busy = false;
>> + atomic64_dec((&pqueue->pending_count));
>> + pentry->post_arg = NULL;
>> + pending_queue_inc_front(pqinfo, qno);
>> + do_request_cleanup(cptvf, info);
>> + spin_unlock_bh(&pqueue->lock);
>> + break;
>> + } else if ((*info->alternate_caddr ==
>> + (~COMPLETION_CODE_INIT)) &&
>> + (info->extra_time < TIME_IN_RESET_COUNT)) {
>> + info->time_in = jiffies;
>> + info->extra_time++;
>> + spin_unlock_bh(&pqueue->lock);
>> + break;
>> + }
>> + }
>> +
>> + pentry->completion_addr = NULL;
>> + pentry->busy = false;
>> + pentry->post_arg = NULL;
>> + atomic64_dec((&pqueue->pending_count));
>> + pending_queue_inc_front(pqinfo, qno);
>> + spin_unlock_bh(&pqueue->lock);
>> +
>> + do_post_process(info->cptvf, info);
>> + /*
>> + * Calling callback after we find
>> + * that the request has been serviced
>> + */
>> + pentry->callback(ccode, pentry->callback_arg);
>> + }
>> +}
>> +
>> +int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
>> +{
>> + int ret = 0, clear = 0, queue = 0;
>> + struct cpt_info_buffer *info = NULL;
>> + struct cptvf_request *cpt_req = NULL;
>> + union ctrl_info *ctrl = NULL;
>> + struct pending_entry *pentry = NULL;
>> + struct pending_queue *pqueue = NULL;
>> + struct pci_dev *pdev = cptvf->pdev;
>> + u8 group = 0;
>> + struct cpt_vq_command vq_cmd;
>> + union cpt_inst_s cptinst;
>> +
>> + if (unlikely(!cptvf || !req)) {
> You already dereferenced cptvf above.
thanks for pointing it out.
>> + dev_err(&pdev->dev, "Invalid inputs (cptvf: %p, req: %p)\n",
>> + cptvf, req);
>> + return -EINVAL;
>> + }
>> +
>> + info = kzalloc(sizeof(*info), GFP_KERNEL | GFP_ATOMIC);
> What do you expect to happen with GFP_KERNEL | GFP_ATOMIC?
>
>> + if (unlikely(!info)) {
>> + dev_err(&pdev->dev, "Unable to allocate memory for info_buffer\n");
>> + return -ENOMEM;
>> + }
>> +
>> + cpt_req = (struct cptvf_request *)&req->req;
>> + ctrl = (union ctrl_info *)&req->ctrl;
>> +
>> + info->cptvf = cptvf;
>> + group = ctrl->s.grp;
>> + ret = setup_sgio_list(cptvf, info, req);
>> + if (ret) {
>> + dev_err(&pdev->dev, "Setting up SG list failed");
>> + goto request_cleanup;
>> + }
>> +
>> + cpt_req->dlen = info->dlen;
>> + /*
>> + * Get buffer for union cpt_res_s response
>> + * structure and its physical address
>> + */
>> + info->completion_addr = kzalloc(sizeof(union cpt_res_s),
>> + GFP_KERNEL | GFP_ATOMIC);
> Same as above, you also never checked if it had failed.
>
>> + *((u8 *)(info->completion_addr)) = COMPLETION_CODE_INIT;
> Supposedly info->completion_addr is a "union cpt_res_s", why do you
> cast it to u8 ptr?
okay will change it.
>> + info->comp_baddr = dma_map_single(&pdev->dev,
>> + (void *)info->completion_addr,
>> + sizeof(union cpt_res_s),
>> + DMA_BIDIRECTIONAL);
>> + if (dma_mapping_error(&pdev->dev, info->comp_baddr)) {
>> + dev_err(&pdev->dev, "mapping compptr Failed %lu\n",
>> + sizeof(union cpt_res_s));
>> + ret = -EFAULT;
>> + goto request_cleanup;
>> + }
>> +
>> + /* Fill the VQ command */
>> + vq_cmd.cmd.u64 = 0;
>> + vq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags);
>> + vq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1);
>> + vq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2);
>> + vq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen);
>> +
>> + /* 64-bit swap for microcode data reads, not needed for addresses*/
>> + vq_cmd.cmd.u64 = cpu_to_be64(vq_cmd.cmd.u64);
>> + vq_cmd.dptr = info->dptr_baddr;
>> + vq_cmd.rptr = info->rptr_baddr;
>> + vq_cmd.cptr.u64 = 0;
>> + vq_cmd.cptr.s.grp = group;
>> + /* Get Pending Entry to submit command */
>> + /* Always queue 0, because 1 queue per VF */
>> + queue = 0;
>> + pqueue = &cptvf->pqinfo.queue[queue];
>> +
>> + if (atomic64_read(&pqueue->pending_count) > PENDING_THOLD) {
>> + dev_err(&pdev->dev, "pending threshold reached\n");
>> + process_pending_queue(cptvf, &cptvf->pqinfo, queue);
>> + }
>> +
>> +get_pending_entry:
>> + spin_lock_bh(&pqueue->lock);
>> + pentry = get_free_pending_entry(pqueue, cptvf->pqinfo.qlen);
>> + if (unlikely(!pentry)) {
>> + spin_unlock_bh(&pqueue->lock);
>> + if (clear == 0) {
>> + process_pending_queue(cptvf, &cptvf->pqinfo, queue);
>> + clear = 1;
>> + goto get_pending_entry;
>> + }
>> + dev_err(&pdev->dev, "Get free entry failed\n");
>> + dev_err(&pdev->dev, "queue: %d, rear: %d, front: %d\n",
>> + queue, pqueue->rear, pqueue->front);
>> + ret = -EFAULT;
>> + goto request_cleanup;
>> + }
>> +
>> + pentry->completion_addr = info->completion_addr;
>> + pentry->post_arg = (void *)info;
>> + pentry->callback = req->callback;
>> + pentry->callback_arg = req->callback_arg;
>> + info->pentry = pentry;
>> + pentry->busy = true;
>> + atomic64_inc(&pqueue->pending_count);
>> +
>> + /* Send CPT command */
>> + info->pentry = pentry;
>> + info->time_in = jiffies;
>> + info->req = req;
>> +
>> + /* Create the CPT_INST_S type command for HW intrepretation */
>> + cptinst.s.doneint = true;
>> + cptinst.s.res_addr = (u64)info->comp_baddr;
>> + cptinst.s.tag = 0;
>> + cptinst.s.grp = 0;
>> + cptinst.s.wq_ptr = 0;
>> + cptinst.s.ei0 = vq_cmd.cmd.u64;
>> + cptinst.s.ei1 = vq_cmd.dptr;
>> + cptinst.s.ei2 = vq_cmd.rptr;
>> + cptinst.s.ei3 = vq_cmd.cptr.u64;
>> +
>> + ret = send_cpt_command(cptvf, &cptinst, queue);
>> + spin_unlock_bh(&pqueue->lock);
>> + if (unlikely(ret)) {
>> + dev_err(&pdev->dev, "Send command failed for AE\n");
>> + ret = -EFAULT;
>> + goto request_cleanup;
>> + }
>> +
>> + return 0;
>> +
>> +request_cleanup:
>> + dev_dbg(&pdev->dev, "Failed to submit CPT command\n");
>> + do_request_cleanup(cptvf, info);
>> +
>> + return ret;
>> +}
>> +
>> +void vq_post_process(struct cpt_vf *cptvf, u32 qno)
>> +{
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (unlikely(qno > cptvf->nr_queues)) {
>> + dev_err(&pdev->dev, "Request for post processing on invalid pending queue: %u\n",
>> + qno);
>> + return;
>> + }
>> +
>> + process_pending_queue(cptvf, &cptvf->pqinfo, qno);
>> +}
>> +
>> +int cptvf_do_request(void *vfdev, struct cpt_request_info *req)
>> +{
>> + struct cpt_vf *cptvf = (struct cpt_vf *)vfdev;
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (!cpt_device_ready(cptvf)) {
>> + dev_err(&pdev->dev, "CPT Device is not ready");
>> + return -ENODEV;
>> + }
>> +
>> + if ((cptvf->vftype == SE_TYPES) && (!req->ctrl.s.se_req)) {
>> + dev_err(&pdev->dev, "CPTVF-%d of SE TYPE got AE request",
>> + cptvf->vfid);
>> + return -EINVAL;
>> + } else if ((cptvf->vftype == AE_TYPES) && (req->ctrl.s.se_req)) {
>> + dev_err(&pdev->dev, "CPTVF-%d of AE TYPE got SE request",
>> + cptvf->vfid);
>> + return -EINVAL;
>> + }
>> +
>> + return process_request(cptvf, req);
>> +}
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox