warcsum/warcsum.c
/*
 * if provided chunk is at beginning of member,
 *    process header then hash payload
 * else hash payload.
 * Used as callback function by inflate member.
 */
if provided chunk is at beginning of member,
process header then hash payload
else hash payload.
Used as callback function by inflate member.
if provided chunk is at beginning of member process header then hash payload else hash payload . Used as callback function by inflate member .
void
process_chunk (z_stream* z, int chunk, void* vp)
{
  struct warcsum_struct* ws = ((struct warcsum_struct*) vp);
  int next_out_length = ws->effective_out - z->avail_out;
  time_t now_parse;
  time (&now_parse);
  int read_bytes = 0;
  switch (chunk)
    {
    case CHUNK_FIRST: 
      read_bytes = process_header (z, vp);
      if (read_bytes != -1)
        {
          ws->response = 1;
        }
      if (ws->response)
        {
          if (ws->recalculate_hash)
            {
              if ((next_out_length - read_bytes) >= 4)
                {
                  hash_update (&z->next_out[read_bytes], ws->computed_hash,
                               next_out_length - read_bytes - 4, ws->hash_ctx);
                  memcpy (ws->last_4, &z->next_out[next_out_length - 4], 4);
                  ws->size_last_4 = 4;
                }
              else
                {
                  memcpy (ws->last_4, &z->next_out[read_bytes],
                          next_out_length - read_bytes);
                  ws->size_last_4 = next_out_length - read_bytes;
                }
            }
        }
      break;
    case CHUNK_MIDDLE: 
      if (ws->response) 
        {
          if (next_out_length >= 4)
            {
              if (ws->recalculate_hash)
                {
                  hash_update (ws->last_4, ws->computed_hash,
                               ws->size_last_4, ws->hash_ctx);
                  hash_update (z->next_out, ws->computed_hash,
                               next_out_length - 4, ws->hash_ctx);
                  memcpy (ws->last_4, &z->next_out[next_out_length - 4], 4);
                  ws->size_last_4 = 4;
                }
            }
          else
            {
              if (ws->recalculate_hash)
                {
                  int to_be_hashed = next_out_length + ws->size_last_4 - 4;
                  hash_update (ws->last_4, ws->computed_hash,
                               to_be_hashed, ws->hash_ctx);
                  memmove (ws->last_4,
                           &ws->last_4[ws->size_last_4 - (4 - next_out_length)],
                           4 - next_out_length);
                  memcpy (&ws->last_4[4 - next_out_length],
                          z->next_out,
                          next_out_length);
                  ws->size_last_4 = 4;
                }
            }
        }
      break;
    case CHUNK_LAST:
      if (ws->response) 
        {
          if (ws->recalculate_hash)
            {
              if (next_out_length >= 4)
                {
                  hash_update (ws->last_4, ws->computed_hash,
                               ws->size_last_4, ws->hash_ctx);
                  hash_update (z->next_out, ws->computed_hash,
                               next_out_length - 4, ws->hash_ctx);
                }
              else
                {
                  [0..n] from previous chunk, 
                  hash_update (ws->last_4, ws->computed_hash,
                               ws->size_last_4 + next_out_length - 4,
                               ws->hash_ctx);
                }
            }
        }
      break;
    case CHUNK_FIRST_LAST:
      read_bytes = process_header (z, vp); 
      if (read_bytes != -1)
        {
          ws->response = 1;
        }
      if (ws->response && (next_out_length - read_bytes <= 4) && ws->args.skip_empty)
        {
          ws->response = 0;
        }
      if (ws->response)
        {
          if (ws->recalculate_hash)
            {
              hash_update (&z->next_out[read_bytes], ws->computed_hash,
                           next_out_length - read_bytes - 4, ws->hash_ctx);
            }
        }
      break;
    }
  time_t then_parse;
  time (&then_parse);
  time_parse += difftime (then_parse, now_parse);
}
void process_chunk ( z_stream * z , int chunk , void * vp ) { struct warcsum_struct * ws = ( ( struct warcsum_struct * ) vp ) ; int next_out_length = ws -> effective_out - z -> avail_out ; time_t now_parse ; time ( & now_parse ) ; int read_bytes = 0 ; switch ( chunk ) { case CHUNK_FIRST : read_bytes = process_header ( z , vp ) ; if ( read_bytes != -1 ) { ws -> response = 1 ; } if ( ws -> response ) { if ( ws -> recalculate_hash ) { if ( ( next_out_length - read_bytes ) >= 4 ) { hash_update ( & z -> next_out [ read_bytes ] , ws -> computed_hash , next_out_length - read_bytes - 4 , ws -> hash_ctx ) ; memcpy ( ws -> last_4 , & z -> next_out [ next_out_length - 4 ] , 4 ) ; ws -> size_last_4 = 4 ; } else { memcpy ( ws -> last_4 , & z -> next_out [ read_bytes ] , next_out_length - read_bytes ) ; ws -> size_last_4 = next_out_length - read_bytes ; } } } break ; case CHUNK_MIDDLE : if ( ws -> response ) { if ( next_out_length >= 4 ) { if ( ws -> recalculate_hash ) { hash_update ( ws -> last_4 , ws -> computed_hash , ws -> size_last_4 , ws -> hash_ctx ) ; hash_update ( z -> next_out , ws -> computed_hash , next_out_length - 4 , ws -> hash_ctx ) ; memcpy ( ws -> last_4 , & z -> next_out [ next_out_length - 4 ] , 4 ) ; ws -> size_last_4 = 4 ; } } else { if ( ws -> recalculate_hash ) { int to_be_hashed = next_out_length + ws -> size_last_4 - 4 ; hash_update ( ws -> last_4 , ws -> computed_hash , to_be_hashed , ws -> hash_ctx ) ; memmove ( ws -> last_4 , & ws -> last_4 [ ws -> size_last_4 - ( 4 - next_out_length ) ] , 4 - next_out_length ) ; memcpy ( & ws -> last_4 [ 4 - next_out_length ] , z -> next_out , next_out_length ) ; ws -> size_last_4 = 4 ; } } } break ; case CHUNK_LAST : if ( ws -> response ) { if ( ws -> recalculate_hash ) { if ( next_out_length >= 4 ) { hash_update ( ws -> last_4 , ws -> computed_hash , ws -> size_last_4 , ws -> hash_ctx ) ; hash_update ( z -> next_out , ws -> computed_hash , next_out_length - 4 , ws -> hash_ctx ) ; } else { hash_update ( ws -> last_4 , ws -> computed_hash , ws -> size_last_4 + next_out_length - 4 , ws -> hash_ctx ) ; } } } break ; case CHUNK_FIRST_LAST : read_bytes = process_header ( z , vp ) ; if ( read_bytes != -1 ) { ws -> response = 1 ; } if ( ws -> response && ( next_out_length - read_bytes <= 4 ) && ws -> args . skip_empty ) { ws -> response = 0 ; } if ( ws -> response ) { if ( ws -> recalculate_hash ) { hash_update ( & z -> next_out [ read_bytes ] , ws -> computed_hash , next_out_length - read_bytes - 4 , ws -> hash_ctx ) ; } } break ; } time_t then_parse ; time ( & then_parse ) ; time_parse += difftime ( then_parse , now_parse ) ; }
if provided chunk is at beginning of member,
    process header then hash payload
 else hash payload.
if provided chunk is at beginning of member process header then hash payload else hash payload .
// mm points to vp with casting for more readable and debuggable code. // next_out_length holds inflated buffer size /* TIME */ /* END OF TIME */ /*
   * Last 4 bytes of a chunk are not hashed with the rest of the chunk to make 
   * sure they are not part of "\r\n\r\n" which is the separator between members 
   * in multi member WARC files.
   */ // first chunk of the member // first process header // if header is valid (warc file, warc response and http response) // if header is valid // if recalculate digest /* 
               * if warc payload length is greater than 4 chars, then save the 
               * last 4 chars in last_4 for handling \r\n\r\n 
               * else save all payload in last_4 and update size_last_4 with the
               * number of chars saved
               */ // neither first nor last chunk of member // if header is processed and member is warc response /* MIDDLE CHUNK:
           * check if  chunk length is >= 4 => hash last_4 
           *      + all member except last 4 bytes
           * o.w. follow comments in else section
           */ // hash last 4 bytes from previous chunk // hash current chunk except for last 4 bytes // length of chars to be hashed from last_4 // hash last_4[0..to_be_hashed] // last_4 <- (4 - next_out_length) chars of last_4 + next_out // push last_4[n..4] back to last_4[0..(4-n)]  // memmove used instead of memcpy  //      due to overlapping source and destination // append current chunk to last4  //     (last_4[0..(4-n)] + current chunk) /*
       * LAST CHUNK:
       * check if next_out_length >= 4, hash last_4 of previous chunk, 
       * then hash current chunk except for last 4 bytes of it "\r\n\r\n"
       * 
       * o.w. hash last_4[0..n] of last chunk, where n in next_out_length
       */ // if header is processed // hash last_4 // hash current chunk [0..(n-4)] where n is next_out_length // hash last_4[0..n] from previous chunk,  // where n is next_out_length + size_last_4 - 4 /* Member fits in 1 chunk
       * process header
       * hash chunk[0..(n-4)] where n is next_out_length
       */ // process header // if header is valid (warc file, warc response and http response) // skip member if empty and argument skip empty is set // hash chunk[0..(n-4)], where n is next_out_length /* TIME */ /* END OF TIME */