crypto/rc4/rc4_enc.c:164: error: INTEGER_OVERFLOW_L1
  (0 - 8):unsigned64.
Showing all 1 steps of the trace


crypto/rc4/rc4_enc.c:164:20: Binary operation: (0 - 8):unsigned64
162.           if (!is_endian.little) { /* BIG-ENDIAN CASE */
163.   # define BESHFT(c)      (((sizeof(RC4_CHUNK)-(c)-1)*8)&(sizeof(RC4_CHUNK)*8-1))
164.               for (; len & (0 - sizeof(RC4_CHUNK)); len -= sizeof(RC4_CHUNK)) {
                          ^
165.                   ichunk = *(RC4_CHUNK *) indata;
166.                   otp = RC4_STEP << BESHFT(0);

void RC4(RC4_KEY *key, size_t len, const unsigned char *indata,
         unsigned char *outdata)
{
    register RC4_INT *d;
    register RC4_INT x, y, tx, ty;
    size_t i;
    x = key->x;
    y = key->y;
    d = key->data;
#if defined(RC4_CHUNK) && !defined(PEDANTIC)
# define RC4_STEP       ( \
                        x=(x+1) &0xff,  \
                        tx=d[x],        \
                        y=(tx+y)&0xff,  \
                        ty=d[y],        \
                        d[y]=tx,        \
                        d[x]=ty,        \
                        (RC4_CHUNK)d[(tx+ty)&0xff]\
                        )
    if ((((size_t)indata & (sizeof(RC4_CHUNK) - 1)) |
         ((size_t)outdata & (sizeof(RC4_CHUNK) - 1))) == 0) {
        RC4_CHUNK ichunk, otp;
        const union {
            long one;
            char little;
        } is_endian = {
            1
        };
        if (!is_endian.little) {
# define BESHFT(c)      (((sizeof(RC4_CHUNK)-(c)-1)*8)&(sizeof(RC4_CHUNK)*8-1))
            for (; len & (0 - sizeof(RC4_CHUNK)); len -= sizeof(RC4_CHUNK)) {
                ichunk = *(RC4_CHUNK *) indata;
                otp = RC4_STEP << BESHFT(0);
                otp |= RC4_STEP << BESHFT(1);
                otp |= RC4_STEP << BESHFT(2);
                otp |= RC4_STEP << BESHFT(3);
                if (sizeof(RC4_CHUNK) == 8) {
                    otp |= RC4_STEP << BESHFT(4);
                    otp |= RC4_STEP << BESHFT(5);
                    otp |= RC4_STEP << BESHFT(6);
                    otp |= RC4_STEP << BESHFT(7);
                }
                *(RC4_CHUNK *) outdata = otp ^ ichunk;
                indata += sizeof(RC4_CHUNK);
                outdata += sizeof(RC4_CHUNK);
            }
            if (len) {
                RC4_CHUNK mask = (RC4_CHUNK) - 1, ochunk;
                ichunk = *(RC4_CHUNK *) indata;
                ochunk = *(RC4_CHUNK *) outdata;
                otp = 0;
                i = BESHFT(0);
                mask <<= (sizeof(RC4_CHUNK) - len) << 3;
                switch (len & (sizeof(RC4_CHUNK) - 1)) {
                case 7:
                    otp = RC4_STEP << i, i -= 8;
                case 6:
                    otp |= RC4_STEP << i, i -= 8;
                case 5:
                    otp |= RC4_STEP << i, i -= 8;
                case 4:
                    otp |= RC4_STEP << i, i -= 8;
                case 3:
                    otp |= RC4_STEP << i, i -= 8;
                case 2:
                    otp |= RC4_STEP << i, i -= 8;
                case 1:
                    otp |= RC4_STEP << i, i -= 8;
                case 0:;
                }
                ochunk &= ~mask;
                ochunk |= (otp ^ ichunk) & mask;
                *(RC4_CHUNK *) outdata = ochunk;
            }
            key->x = x;
            key->y = y;
            return;
        } else {
# define LESHFT(c)      (((c)*8)&(sizeof(RC4_CHUNK)*8-1))
            for (; len & (0 - sizeof(RC4_CHUNK)); len -= sizeof(RC4_CHUNK)) {
                ichunk = *(RC4_CHUNK *) indata;
                otp = RC4_STEP;
                otp |= RC4_STEP << 8;
                otp |= RC4_STEP << 16;
                otp |= RC4_STEP << 24;
                if (sizeof(RC4_CHUNK) == 8) {
                    otp |= RC4_STEP << LESHFT(4);
                    otp |= RC4_STEP << LESHFT(5);
                    otp |= RC4_STEP << LESHFT(6);
                    otp |= RC4_STEP << LESHFT(7);
                }
                *(RC4_CHUNK *) outdata = otp ^ ichunk;
                indata += sizeof(RC4_CHUNK);
                outdata += sizeof(RC4_CHUNK);
            }
            if (len) {
                RC4_CHUNK mask = (RC4_CHUNK) - 1, ochunk;
                ichunk = *(RC4_CHUNK *) indata;
                ochunk = *(RC4_CHUNK *) outdata;
                otp = 0;
                i = 0;
                mask >>= (sizeof(RC4_CHUNK) - len) << 3;
                switch (len & (sizeof(RC4_CHUNK) - 1)) {
                case 7:
                    otp = RC4_STEP, i += 8;
                case 6:
                    otp |= RC4_STEP << i, i += 8;
                case 5:
                    otp |= RC4_STEP << i, i += 8;
                case 4:
                    otp |= RC4_STEP << i, i += 8;
                case 3:
                    otp |= RC4_STEP << i, i += 8;
                case 2:
                    otp |= RC4_STEP << i, i += 8;
                case 1:
                    otp |= RC4_STEP << i, i += 8;
                case 0:;
                }
                ochunk &= ~mask;
                ochunk |= (otp ^ ichunk) & mask;
                *(RC4_CHUNK *) outdata = ochunk;
            }
            key->x = x;
            key->y = y;
            return;
        }
    }
#endif
#define LOOP(in,out) \
                x=((x+1)&0xff); \
                tx=d[x]; \
                y=(tx+y)&0xff; \
                d[x]=ty=d[y]; \
                d[y]=tx; \
                (out) = d[(tx+ty)&0xff]^ (in);
#ifndef RC4_INDEX
# define RC4_LOOP(a,b,i) LOOP(*((a)++),*((b)++))
#else
# define RC4_LOOP(a,b,i) LOOP(a[i],b[i])
#endif
    i = len >> 3;
    if (i) {
        for (;;) {
            RC4_LOOP(indata, outdata, 0);
            RC4_LOOP(indata, outdata, 1);
            RC4_LOOP(indata, outdata, 2);
            RC4_LOOP(indata, outdata, 3);
            RC4_LOOP(indata, outdata, 4);
            RC4_LOOP(indata, outdata, 5);
            RC4_LOOP(indata, outdata, 6);
            RC4_LOOP(indata, outdata, 7);
#ifdef RC4_INDEX
            indata += 8;
            outdata += 8;
#endif
            if (--i == 0)
                break;
        }
    }
    i = len & 0x07;
    if (i) {
        for (;;) {
            RC4_LOOP(indata, outdata, 0);
            if (--i == 0)
                break;
            RC4_LOOP(indata, outdata, 1);
            if (--i == 0)
                break;
            RC4_LOOP(indata, outdata, 2);
            if (--i == 0)
                break;
            RC4_LOOP(indata, outdata, 3);
            if (--i == 0)
                break;
            RC4_LOOP(indata, outdata, 4);
            if (--i == 0)
                break;
            RC4_LOOP(indata, outdata, 5);
            if (--i == 0)
                break;
            RC4_LOOP(indata, outdata, 6);
            if (--i == 0)
                break;
        }
    }
    key->x = x;
    key->y = y;
}
['void RC4(RC4_KEY *key, size_t len, const unsigned char *indata,\n         unsigned char *outdata)\n{\n    register RC4_INT *d;\n    register RC4_INT x, y, tx, ty;\n    size_t i;\n    x = key->x;\n    y = key->y;\n    d = key->data;\n#if defined(RC4_CHUNK) && !defined(PEDANTIC)\n# define RC4_STEP       ( \\\n                        x=(x+1) &0xff,  \\\n                        tx=d[x],        \\\n                        y=(tx+y)&0xff,  \\\n                        ty=d[y],        \\\n                        d[y]=tx,        \\\n                        d[x]=ty,        \\\n                        (RC4_CHUNK)d[(tx+ty)&0xff]\\\n                        )\n    if ((((size_t)indata & (sizeof(RC4_CHUNK) - 1)) |\n         ((size_t)outdata & (sizeof(RC4_CHUNK) - 1))) == 0) {\n        RC4_CHUNK ichunk, otp;\n        const union {\n            long one;\n            char little;\n        } is_endian = {\n            1\n        };\n        if (!is_endian.little) {\n# define BESHFT(c)      (((sizeof(RC4_CHUNK)-(c)-1)*8)&(sizeof(RC4_CHUNK)*8-1))\n            for (; len & (0 - sizeof(RC4_CHUNK)); len -= sizeof(RC4_CHUNK)) {\n                ichunk = *(RC4_CHUNK *) indata;\n                otp = RC4_STEP << BESHFT(0);\n                otp |= RC4_STEP << BESHFT(1);\n                otp |= RC4_STEP << BESHFT(2);\n                otp |= RC4_STEP << BESHFT(3);\n                if (sizeof(RC4_CHUNK) == 8) {\n                    otp |= RC4_STEP << BESHFT(4);\n                    otp |= RC4_STEP << BESHFT(5);\n                    otp |= RC4_STEP << BESHFT(6);\n                    otp |= RC4_STEP << BESHFT(7);\n                }\n                *(RC4_CHUNK *) outdata = otp ^ ichunk;\n                indata += sizeof(RC4_CHUNK);\n                outdata += sizeof(RC4_CHUNK);\n            }\n            if (len) {\n                RC4_CHUNK mask = (RC4_CHUNK) - 1, ochunk;\n                ichunk = *(RC4_CHUNK *) indata;\n                ochunk = *(RC4_CHUNK *) outdata;\n                otp = 0;\n                i = BESHFT(0);\n                mask <<= (sizeof(RC4_CHUNK) - len) << 3;\n                switch (len & (sizeof(RC4_CHUNK) - 1)) {\n                case 7:\n                    otp = RC4_STEP << i, i -= 8;\n                case 6:\n                    otp |= RC4_STEP << i, i -= 8;\n                case 5:\n                    otp |= RC4_STEP << i, i -= 8;\n                case 4:\n                    otp |= RC4_STEP << i, i -= 8;\n                case 3:\n                    otp |= RC4_STEP << i, i -= 8;\n                case 2:\n                    otp |= RC4_STEP << i, i -= 8;\n                case 1:\n                    otp |= RC4_STEP << i, i -= 8;\n                case 0:;\n                }\n                ochunk &= ~mask;\n                ochunk |= (otp ^ ichunk) & mask;\n                *(RC4_CHUNK *) outdata = ochunk;\n            }\n            key->x = x;\n            key->y = y;\n            return;\n        } else {\n# define LESHFT(c)      (((c)*8)&(sizeof(RC4_CHUNK)*8-1))\n            for (; len & (0 - sizeof(RC4_CHUNK)); len -= sizeof(RC4_CHUNK)) {\n                ichunk = *(RC4_CHUNK *) indata;\n                otp = RC4_STEP;\n                otp |= RC4_STEP << 8;\n                otp |= RC4_STEP << 16;\n                otp |= RC4_STEP << 24;\n                if (sizeof(RC4_CHUNK) == 8) {\n                    otp |= RC4_STEP << LESHFT(4);\n                    otp |= RC4_STEP << LESHFT(5);\n                    otp |= RC4_STEP << LESHFT(6);\n                    otp |= RC4_STEP << LESHFT(7);\n                }\n                *(RC4_CHUNK *) outdata = otp ^ ichunk;\n                indata += sizeof(RC4_CHUNK);\n                outdata += sizeof(RC4_CHUNK);\n            }\n            if (len) {\n                RC4_CHUNK mask = (RC4_CHUNK) - 1, ochunk;\n                ichunk = *(RC4_CHUNK *) indata;\n                ochunk = *(RC4_CHUNK *) outdata;\n                otp = 0;\n                i = 0;\n                mask >>= (sizeof(RC4_CHUNK) - len) << 3;\n                switch (len & (sizeof(RC4_CHUNK) - 1)) {\n                case 7:\n                    otp = RC4_STEP, i += 8;\n                case 6:\n                    otp |= RC4_STEP << i, i += 8;\n                case 5:\n                    otp |= RC4_STEP << i, i += 8;\n                case 4:\n                    otp |= RC4_STEP << i, i += 8;\n                case 3:\n                    otp |= RC4_STEP << i, i += 8;\n                case 2:\n                    otp |= RC4_STEP << i, i += 8;\n                case 1:\n                    otp |= RC4_STEP << i, i += 8;\n                case 0:;\n                }\n                ochunk &= ~mask;\n                ochunk |= (otp ^ ichunk) & mask;\n                *(RC4_CHUNK *) outdata = ochunk;\n            }\n            key->x = x;\n            key->y = y;\n            return;\n        }\n    }\n#endif\n#define LOOP(in,out) \\\n                x=((x+1)&0xff); \\\n                tx=d[x]; \\\n                y=(tx+y)&0xff; \\\n                d[x]=ty=d[y]; \\\n                d[y]=tx; \\\n                (out) = d[(tx+ty)&0xff]^ (in);\n#ifndef RC4_INDEX\n# define RC4_LOOP(a,b,i) LOOP(*((a)++),*((b)++))\n#else\n# define RC4_LOOP(a,b,i) LOOP(a[i],b[i])\n#endif\n    i = len >> 3;\n    if (i) {\n        for (;;) {\n            RC4_LOOP(indata, outdata, 0);\n            RC4_LOOP(indata, outdata, 1);\n            RC4_LOOP(indata, outdata, 2);\n            RC4_LOOP(indata, outdata, 3);\n            RC4_LOOP(indata, outdata, 4);\n            RC4_LOOP(indata, outdata, 5);\n            RC4_LOOP(indata, outdata, 6);\n            RC4_LOOP(indata, outdata, 7);\n#ifdef RC4_INDEX\n            indata += 8;\n            outdata += 8;\n#endif\n            if (--i == 0)\n                break;\n        }\n    }\n    i = len & 0x07;\n    if (i) {\n        for (;;) {\n            RC4_LOOP(indata, outdata, 0);\n            if (--i == 0)\n                break;\n            RC4_LOOP(indata, outdata, 1);\n            if (--i == 0)\n                break;\n            RC4_LOOP(indata, outdata, 2);\n            if (--i == 0)\n                break;\n            RC4_LOOP(indata, outdata, 3);\n            if (--i == 0)\n                break;\n            RC4_LOOP(indata, outdata, 4);\n            if (--i == 0)\n                break;\n            RC4_LOOP(indata, outdata, 5);\n            if (--i == 0)\n                break;\n            RC4_LOOP(indata, outdata, 6);\n            if (--i == 0)\n                break;\n        }\n    }\n    key->x = x;\n    key->y = y;\n}']