warcsum/warcsum.c
/* Processes next member from warc.gz file pointer */
Processes next member from warc.gz file pointer
Processes next member from warc . gz file pointer
int
process_member (FILE* f_in, FILE* f_out, z_stream *z,
                struct warcsum_struct* ws)
{
  if (ws->args.verbose == 2)
    {
      fprintf (stderr, "\n\n");
      fprintf (stderr, "OFFSET: %ld\n", ftello(f_in));
    }
  ws->response = 0;
  ws->last_4[0] = '\0';
  ws->START = ftello(f_in);
  inflateReset2 (z, 31);
  hash_init (&ws->hash_ctx, ws->args.hash_code);
  int err = inflateMember (z, f_in, ws->effective_in,
                           ws->effective_out, process_chunk, ws);
  hash_final (ws->hash_ctx, ws->args.hash_code,
              ws->computed_digest, ws->args);
  ws->END = ftello(f_in);
  if (ws->response) 
    {
      char final_digest[DIGEST_LENGTH];
      char final_hash[HASH_LENGTH];
      if (ws->recalculate_hash)
        {
          strcpy (final_digest, ws->computed_digest);
        }
      else 
        {
          strcpy (final_digest, ws->stored_digest);
        }
      strcpy (final_hash, ws->args.hash_char);
      fprintf(f_out, "%s %lu %lu %s %s %s:%s\n", ws->WARCFILE_NAME,
              ws->START, ws->END - ws->START, ws->URI,
              ws->DATE, final_hash, final_digest);
      if (ws->args.verbose == 2) {
          fprintf(stderr, "%s %lu %lu %s %s %s:%s\n", ws->WARCFILE_NAME,
                  ws->START, ws->END - ws->START, ws->URI,
                  ws->DATE, final_hash, final_digest);
      }
      free (ws->URI);
      return 0;
    }
  else
    {
      free (ws->URI);
      return 1;
    }
}
int process_member ( FILE * f_in , FILE * f_out , z_stream * z , struct warcsum_struct * ws ) { if ( ws -> args . verbose == 2 ) { fprintf ( stderr , " \n \n " ) ; fprintf ( stderr , " \n " , ftello ( f_in ) ) ; } ws -> response = 0 ; ws -> last_4 [ 0 ] = ' \0 ' ; ws -> START = ftello ( f_in ) ; inflateReset2 ( z , 31 ) ; hash_init ( & ws -> hash_ctx , ws -> args . hash_code ) ; int err = inflateMember ( z , f_in , ws -> effective_in , ws -> effective_out , process_chunk , ws ) ; hash_final ( ws -> hash_ctx , ws -> args . hash_code , ws -> computed_digest , ws -> args ) ; ws -> END = ftello ( f_in ) ; if ( ws -> response ) { char final_digest [ DIGEST_LENGTH ] ; char final_hash [ HASH_LENGTH ] ; if ( ws -> recalculate_hash ) { strcpy ( final_digest , ws -> computed_digest ) ; } else { strcpy ( final_digest , ws -> stored_digest ) ; } strcpy ( final_hash , ws -> args . hash_char ) ; fprintf ( f_out , " \n " , ws -> WARCFILE_NAME , ws -> START , ws -> END - ws -> START , ws -> URI , ws -> DATE , final_hash , final_digest ) ; if ( ws -> args . verbose == 2 ) { fprintf ( stderr , " \n " , ws -> WARCFILE_NAME , ws -> START , ws -> END - ws -> START , ws -> URI , ws -> DATE , final_hash , final_digest ) ; } free ( ws -> URI ) ; return 0 ; } else { free ( ws -> URI ) ; return 1 ; } }
Processes next member from warc.gz file pointer
Processes next member from warc . gz file pointer
/* Reset mydata */ /* Reset z_stream */ // for notes on '31' refer to libgzmulti code /* Initialize hash_ctx struct */ /* call inflateMember from libgzMulti */ /* Finalize hash_ctx and produce calculated digest */ /* store position of end of member in compressed file
   * requested by warccolres to download member
   */ // if processed member was response // if calculated hash was chosen // if stored hash was chosen // Write to digests file