lib/millstone.js
// Unzip function, geared specifically toward unpacking a shapefile.
Unzip function, geared specifically toward unpacking a shapefile.
Unzip function geared specifically toward unpacking a shapefile .
function unzip(file, callback) {
    var metafile = metapath(file);
    try {
        var meta_data = fs.readFileSync(metafile);
        if (meta_data) {
            var meta = JSON.parse(meta_data);
            var dest_file = meta['unzipped_file'];
            if (dest_file && existsSync(dest_file)) {
                if (env == 'development') console.error('[millstone] found previously unzipped file: ' + dest_file);
                return callback(null,dest_file);
            }
        } else {
            if (env == 'development') console.error('[millstone] empty meta file for zipfile: ' + metafile);
        }
    }
    catch (err) {
        if (env == 'development') console.error('[millstone] error opening meta file for zipfile: ' + err.message);
    }
    var zf;
    try {
        zf = new zipfile.ZipFile(file);
    } catch (err) {
        return callback(err);
    }
    if (zf.names.length < 1) {
        return callback(new Error("could not find any valid data in zip archive: '" + file + "'"));
    }
    var remain = zf.names.length;
    var ds_files = _(zf.names).chain()
        .reject(function(name) {
            return (name && (name[0] === '.' || path.basename(name)[0] === '.'));
        })
        .map(function(name) {
            if (!valid_ds_extensions[path.extname(name).toLowerCase()]) return;
            var new_name = path.join(
                path.dirname(file),
                path.basename(file, path.extname(file)) +
                path.extname(name).toLowerCase());
            return {new_name:new_name,original_name:name};
        })
        .uniq()
        .compact()
        .value();
    if (!ds_files || ds_files.length < 1) return callback(new Error("Valid datasource not detected (by extension) in zip: '" + file + "'"));
    var original_name = ds_files[0].original_name;
    var new_name = ds_files[0].new_name;
    var len = Object.keys(ds_files).length;
    if (len > 1) {
        for (var i=0;i<len;++i) {
            var fname = ds_files[i].original_name;
            if (path.extname(fname) == '.shp') {
                original_name = fname;
                new_name = ds_files[i].new_name;
                break;
            }
        }
        if (env == 'development') {
            console.warn('[millstone] warning: detected more than one file in zip (by ext) that may be valid: ');
            for (var i=0;i<len;++i) {
                console.warn('[millstone]   ' + ds_files[i].original_name);
            }
            console.warn('[millstone] picked: ' + original_name);
        }
    }
    if (!new_name) return callback(new Error("Valid datasource not detected (by extension) in zip: '" + file + "'"));
    if (env == 'development') {
        console.warn('[millstone] renaming ' + original_name + ' to ' + new_name);
    }
    var search_basename = path.basename(original_name,path.extname(original_name));
    zf.names.forEach(function(name) {
        var basename = path.basename(name);
        if (!path.extname(name) || (name.indexOf(search_basename) < 0) || name[0] === '.' || basename[0] === '.') {
            remain--;
            if (!remain) callback(null, new_name);
        } else {
            var dest = path.join(
                path.dirname(file),
                path.basename(file, path.extname(file)) +
                path.extname(name).toLowerCase()
            );
            zf.copyFile(name, dest, function(err) {
                if (err) return callback(err);
                remain--;
                if (!remain) {
                    add_item_to_metafile(metafile,'unzipped_file',new_name,function(err) {
                        if (err) throw err;
                        callback(null, new_name);
                    });
                }
            });
        }
    });
}
function unzip ( file , callback ) { var metafile = metapath ( file ) ; try { var meta_data = fs . readFileSync ( metafile ) ; if ( meta_data ) { var meta = JSON . parse ( meta_data ) ; var dest_file = meta [ 'unzipped_file' ] ; if ( dest_file && existsSync ( dest_file ) ) { if ( env == 'development' ) console . error ( '[millstone] found previously unzipped file: ' + dest_file ) ; return callback ( null , dest_file ) ; } } else { if ( env == 'development' ) console . error ( '[millstone] empty meta file for zipfile: ' + metafile ) ; } } catch ( err ) { if ( env == 'development' ) console . error ( '[millstone] error opening meta file for zipfile: ' + err . message ) ; } var zf ; try { zf = new zipfile . ZipFile ( file ) ; } catch ( err ) { return callback ( err ) ; } if ( zf . names . length < 1 ) { return callback ( new Error ( "could not find any valid data in zip archive: '" + file + "'" ) ) ; } var remain = zf . names . length ; var ds_files = _ ( zf . names ) . chain ( ) . reject ( function ( name ) { return ( name && ( name [ 0 ] === '.' || path . basename ( name ) [ 0 ] === '.' ) ) ; } ) . map ( function ( name ) { if ( ! valid_ds_extensions [ path . extname ( name ) . toLowerCase ( ) ] ) return ; var new_name = path . join ( path . dirname ( file ) , path . basename ( file , path . extname ( file ) ) + path . extname ( name ) . toLowerCase ( ) ) ; return { new_name : new_name , original_name : name } ; } ) . uniq ( ) . compact ( ) . value ( ) ; if ( ! ds_files || ds_files . length < 1 ) return callback ( new Error ( "Valid datasource not detected (by extension) in zip: '" + file + "'" ) ) ; var original_name = ds_files [ 0 ] . original_name ; var new_name = ds_files [ 0 ] . new_name ; var len = Object . keys ( ds_files ) . length ; if ( len > 1 ) { for ( var i = 0 ; i < len ; ++ i ) { var fname = ds_files [ i ] . original_name ; if ( path . extname ( fname ) == '.shp' ) { original_name = fname ; new_name = ds_files [ i ] . new_name ; break ; } } if ( env == 'development' ) { console . warn ( '[millstone] warning: detected more than one file in zip (by ext) that may be valid: ' ) ; for ( var i = 0 ; i < len ; ++ i ) { console . warn ( '[millstone]   ' + ds_files [ i ] . original_name ) ; } console . warn ( '[millstone] picked: ' + original_name ) ; } } if ( ! new_name ) return callback ( new Error ( "Valid datasource not detected (by extension) in zip: '" + file + "'" ) ) ; if ( env == 'development' ) { console . warn ( '[millstone] renaming ' + original_name + ' to ' + new_name ) ; } var search_basename = path . basename ( original_name , path . extname ( original_name ) ) ; zf . names . forEach ( function ( name ) { var basename = path . basename ( name ) ; if ( ! path . extname ( name ) || ( name . indexOf ( search_basename ) < 0 ) || name [ 0 ] === '.' || basename [ 0 ] === '.' ) { remain -- ; if ( ! remain ) callback ( null , new_name ) ; } else { var dest = path . join ( path . dirname ( file ) , path . basename ( file , path . extname ( file ) ) + path . extname ( name ) . toLowerCase ( ) ) ; zf . copyFile ( name , dest , function ( err ) { if ( err ) return callback ( err ) ; remain -- ; if ( ! remain ) { add_item_to_metafile ( metafile , 'unzipped_file' , new_name , function ( err ) { if ( err ) throw err ; callback ( null , new_name ) ; } ) ; } } ) ; } } ) ; }
Unzip function, geared specifically toward unpacking a shapefile.
Unzip function geared specifically toward unpacking a shapefile .
// return cached result of unzipped file // intentionally sync here to avoid race condition unzipping // same file for multiple layers // prefer first .shp // only unzip files that match our target name // naive '(name.indexOf(search_basename) < 0)' does the trick // yes this is simplistic, but its better than corrupt data: https://github.com/mapbox/millstone/issues/99 // Skip directories, hiddens. // We're brutal in our expectations -- don't support nested // directories, and rename any file from `arbitraryName.SHP` // to `[hash].shp`. // ignore error from add_item_to_metafile //if (err && env == 'development') console.error('[millstone] ' + err.message);