def write_vector_window(
    in_data=None, out_schema=None, out_tile=None, out_path=None, bucket_resource=None
):
    """
    Write features to GeoJSON file.

    Parameters
    ----------
    in_data : features
    out_schema : dictionary
        output schema for fiona
    out_tile : ``BufferedTile``
        tile used for output extent
    out_path : string
        output path for GeoJSON file
    """
    # Delete existing file.
    try:
        os.remove(out_path)
    except OSError:
        pass

    out_features = []
    for feature in in_data:
        try:
            # clip feature geometry to tile bounding box and append for writing
            # if clipped feature still
            for out_geom in multipart_to_singleparts(
                clean_geometry_type(
                    to_shape(feature["geometry"]).intersection(out_tile.bbox),
                    out_schema["geometry"]
                )
            ):
                out_features.append({
                    "geometry": mapping(out_geom),
                    "properties": feature["properties"]
                })
        except Exception as e:
            logger.warning("failed to prepare geometry for writing: %s", e)
            continue

    # write if there are output features
    if out_features:

        try:
            if out_path.startswith("s3://"):
                # write data to remote file
                with VectorWindowMemoryFile(
                    tile=out_tile,
                    features=out_features,
                    schema=out_schema,
                    driver="GeoJSON"
                ) as memfile:
                    logger.debug((out_tile.id, "upload tile", out_path))
                    bucket_resource.put_object(
                        Key="/".join(out_path.split("/")[3:]),
                        Body=memfile
                    )
            else:
                # write data to local file
                with fiona.open(
                    out_path, 'w', schema=out_schema, driver="GeoJSON",
                    crs=out_tile.crs.to_dict()
                ) as dst:
                    logger.debug((out_tile.id, "write tile", out_path))
                    dst.writerecords(out_features)
        except Exception as e:
            logger.error("error while writing file %s: %s", out_path, e)
            raise

    else:
        logger.debug((out_tile.id, "nothing to write", out_path))


def clean_geometry_type(geometry, target_type, allow_multipart=True):
    """
    Return geometry of a specific type if possible.

    Filters and splits up GeometryCollection into target types. This is
    necessary when after clipping and/or reprojecting the geometry types from
    source geometries change (i.e. a Polygon becomes a LineString or a
    LineString becomes Point) in some edge cases.

    Parameters
    ----------
    geometry : ``shapely.geometry``
    target_type : string
        target geometry type
    allow_multipart : bool
        allow multipart geometries (default: True)

    Returns
    -------
    cleaned geometry : ``shapely.geometry``
        returns None if input geometry type differs from target type

    Raises
    ------
    GeometryTypeError : if geometry type does not match target_type
    """
    multipart_geoms = {
        "Point": MultiPoint,
        "LineString": MultiLineString,
        "Polygon": MultiPolygon,
        "MultiPoint": MultiPoint,
        "MultiLineString": MultiLineString,
        "MultiPolygon": MultiPolygon
    }

    if target_type not in multipart_geoms.keys():
        raise TypeError("target type is not supported: %s" % target_type)

    if geometry.geom_type == target_type:
        return geometry

    elif allow_multipart:
        target_multipart_type = multipart_geoms[target_type]
        if geometry.geom_type == "GeometryCollection":
            return target_multipart_type([
                clean_geometry_type(g, target_type, allow_multipart)
                for g in geometry])
        elif any([
            isinstance(geometry, target_multipart_type),
            multipart_geoms[geometry.geom_type] == target_multipart_type
        ]):
            return geometry

    raise GeometryTypeError(
        "geometry type does not match: %s, %s" % (geometry.geom_type, target_type)
    )

def
clean_geometry_type
(
geometry
,
target_type
,
allow_multipart
=
True
)
:
multipart_geoms
=
{
"Point"
:
MultiPoint
,
"LineString"
:
MultiLineString
,
"Polygon"
:
MultiPolygon
,
"MultiPoint"
:
MultiPoint
,
"MultiLineString"
:
MultiLineString
,
"MultiPolygon"
:
MultiPolygon
}
if
target_type
not
in
multipart_geoms
.
keys
(
)
:
raise
TypeError
(
"target type is not supported: %s"
%
target_type
)
if
geometry
.
geom_type
==
target_type
:
return
geometry
elif
allow_multipart
:
target_multipart_type
=
multipart_geoms
[
target_type
]
if
geometry
.
geom_type
==
"GeometryCollection"
:
return
target_multipart_type
(
[
clean_geometry_type
(
g
,
target_type
,
allow_multipart
)
for
g
in
geometry
]
)
elif
any
(
[
isinstance
(
geometry
,
target_multipart_type
)
,
multipart_geoms
[
geometry
.
geom_type
]
==
target_multipart_type
]
)
:
return
geometry
raise
GeometryTypeError
(
"geometry type does not match: %s, %s"
%
(
geometry
.
geom_type
,
target_type
)
)

Return geometry of a specific type if possible .


def multipart_to_singleparts(geom):
    """
    Yield single part geometries if geom is multipart, otherwise yield geom.

    Parameters:
    -----------
    geom : shapely geometry

    Returns:
    --------
    shapely single part geometries
    """
    if isinstance(geom, base.BaseGeometry):
        if hasattr(geom, "geoms"):
            for subgeom in geom:
                yield subgeom
        else:
            yield geom

def multipart_to_singleparts ( geom ) : if isinstance ( geom , base . BaseGeometry ) : if hasattr ( geom , "geoms" ) : for subgeom in geom : yield subgeom else : yield geom

Yield single part geometries if geom is multipart otherwise yield geom .


def execute(
    mp,
    td_resampling="nearest",
    td_matching_method="gdal",
    td_matching_max_zoom=None,
    td_matching_precision=8,
    td_fallback_to_higher_zoom=False,
    clip_pixelbuffer=0,
    **kwargs
):
    """
    Convert and optionally clip input raster data.

    Inputs:
    -------
    raster
        singleband or multiband data input
    clip (optional)
        vector data used to clip output

    Parameters
    ----------
    td_resampling : str (default: 'nearest')
        Resampling used when reading from TileDirectory.
    td_matching_method : str ('gdal' or 'min') (default: 'gdal')
        gdal: Uses GDAL's standard method. Here, the target resolution is
            calculated by averaging the extent's pixel sizes over both x and y
            axes. This approach returns a zoom level which may not have the
            best quality but will speed up reading significantly.
        min: Returns the zoom level which matches the minimum resolution of the
            extents four corner pixels. This approach returns the zoom level
            with the best possible quality but with low performance. If the
            tile extent is outside of the destination pyramid, a
            TopologicalError will be raised.
    td_matching_max_zoom : int (optional, default: None)
        If set, it will prevent reading from zoom levels above the maximum.
    td_matching_precision : int (default: 8)
        Round resolutions to n digits before comparing.
    td_fallback_to_higher_zoom : bool (default: False)
        In case no data is found at zoom level, try to read data from higher
        zoom levels. Enabling this setting can lead to many IO requests in
        areas with no data.
    clip_pixelbuffer : int
        Use pixelbuffer when clipping output by geometry. (default: 0)

    Output
    ------
    np.ndarray
    """
    # read clip geometry
    if "clip" in mp.params["input"]:
        clip_geom = mp.open("clip").read()
        if not clip_geom:
            logger.debug("no clip data over tile")
            return "empty"
    else:
        clip_geom = []

    with mp.open(
        "raster",
        matching_method=td_matching_method,
        matching_max_zoom=td_matching_max_zoom,
        matching_precision=td_matching_precision,
        fallback_to_higher_zoom=td_fallback_to_higher_zoom,
        resampling=td_resampling
    ) as raster:
        raster_data = raster.read()
        if raster.is_empty() or raster_data[0].mask.all():
            logger.debug("raster empty")
            return "empty"

    if clip_geom:
        # apply original nodata mask and clip
        clipped = mp.clip(
            np.where(raster_data[0].mask, mp.params["output"].nodata, raster_data),
            clip_geom,
            clip_buffer=clip_pixelbuffer,
            inverted=True
        )
        return np.where(clipped.mask, clipped, mp.params["output"].nodata)
    else:
        return np.where(raster_data[0].mask, mp.params["output"].nodata, raster_data)

def
execute
(
mp
,
td_resampling
=
"nearest"
,
td_matching_method
=
"gdal"
,
td_matching_max_zoom
=
,
td_matching_precision
=
8
,
td_fallback_to_higher_zoom
=
False
,
clip_pixelbuffer
=
0
,
*
*
kwargs
)
:
# read clip geometry
if
"clip"
in
mp
.
params
[
"input"
]
:
clip_geom
=
mp
.
open
(
"clip"
)
.
read
(
)
if
not
clip_geom
:
logger
.
debug
(
"no clip data over tile"
)
return
"empty"
else
:
clip_geom
=
[
]
with
mp
.
open
(
"raster"
,
matching_method
=
td_matching_method
,
matching_max_zoom
=
td_matching_max_zoom
,
matching_precision
=
td_matching_precision
,
fallback_to_higher_zoom
=
td_fallback_to_higher_zoom
,
resampling
=
td_resampling
)
as
raster
:
raster_data
=
raster
.
read
(
)
if
raster
.
is_empty
(
)
or
raster_data
[
0
]
.
mask
.
all
(
)
:
logger
.
debug
(
"raster empty"
)
return
"empty"
if
clip_geom
:
# apply original nodata mask and clip
clipped
=
mp
.
clip
(
np
.
where
(
raster_data
[
0
]
.
mask
,
mp
.
params
[
"output"
]
.
nodata
,
raster_data
)
,
clip_geom
,
clip_buffer
=
clip_pixelbuffer
,
inverted
=
True
)
return
np
.
where
(
clipped
.
mask
,
clipped
,
mp
.
params
[
"output"
]
.
nodata
)
else
:
return
np
.
where
(
raster_data
[
0
]
.
mask
,
mp
.
params
[
"output"
]
.
nodata
,
raster_data
)

Convert and optionally clip input raster data .


def get_best_zoom_level(input_file, tile_pyramid_type):
    """
    Determine the best base zoom level for a raster.

    "Best" means the maximum zoom level where no oversampling has to be done.

    Parameters
    ----------
    input_file : path to raster file
    tile_pyramid_type : ``TilePyramid`` projection (``geodetic`` or``mercator``)

    Returns
    -------
    zoom : integer
    """
    tile_pyramid = BufferedTilePyramid(tile_pyramid_type)
    with rasterio.open(input_file, "r") as src:
        xmin, ymin, xmax, ymax = reproject_geometry(
            segmentize_geometry(
                box(
                    src.bounds.left, src.bounds.bottom, src.bounds.right,
                    src.bounds.top
                ),
                get_segmentize_value(input_file, tile_pyramid)
            ),
            src_crs=src.crs, dst_crs=tile_pyramid.crs
        ).bounds
        x_dif = xmax - xmin
        y_dif = ymax - ymin
        size = float(src.width + src.height)
        avg_resolution = (
            (x_dif / float(src.width)) * (float(src.width) / size) +
            (y_dif / float(src.height)) * (float(src.height) / size)
        )

    for zoom in range(0, 40):
        if tile_pyramid.pixel_x_size(zoom) <= avg_resolution:
            return zoom-1

def get_best_zoom_level ( input_file , tile_pyramid_type ) : tile_pyramid = BufferedTilePyramid ( tile_pyramid_type ) with rasterio . open ( input_file , "r" ) as src : xmin , ymin , xmax , ymax = reproject_geometry ( segmentize_geometry ( box ( src . bounds . left , src . bounds . bottom , src . bounds . right , src . bounds . top ) , get_segmentize_value ( input_file , tile_pyramid ) ) , src_crs = src . crs , dst_crs = tile_pyramid . crs ) . bounds x_dif = xmax - xmin y_dif = ymax - ymin size = float ( src . width + src . height ) avg_resolution = ( ( x_dif / float ( src . width ) ) * ( float ( src . width ) / size ) + ( y_dif / float ( src . height ) ) * ( float ( src . height ) / size ) ) for zoom in range ( 0 , 40 ) : if tile_pyramid . pixel_x_size ( zoom ) <= avg_resolution : return zoom - 1

Determine the best base zoom level for a raster .


def tile_to_zoom_level(tile, dst_pyramid=None, matching_method="gdal", precision=8):
    """
    Determine the best zoom level in target TilePyramid from given Tile.

    Parameters
    ----------
    tile : BufferedTile
    dst_pyramid : BufferedTilePyramid
    matching_method : str ('gdal' or 'min')
        gdal: Uses GDAL's standard method. Here, the target resolution is calculated by
            averaging the extent's pixel sizes over both x and y axes. This approach
            returns a zoom level which may not have the best quality but will speed up
            reading significantly.
        min: Returns the zoom level which matches the minimum resolution of the extent's
            four corner pixels. This approach returns the zoom level with the best
            possible quality but with low performance. If the tile extent is outside of
            the destination pyramid, a TopologicalError will be raised.
    precision : int
        Round resolutions to n digits before comparing.

    Returns
    -------
    zoom : int
    """
    def width_height(bounds):
        try:
            l, b, r, t = reproject_geometry(
                box(*bounds), src_crs=tile.crs, dst_crs=dst_pyramid.crs
            ).bounds
        except ValueError:
            raise TopologicalError("bounds cannot be translated into target CRS")
        return r - l, t - b

    if tile.tp.crs == dst_pyramid.crs:
        return tile.zoom
    else:
        if matching_method == "gdal":
            # use rasterio/GDAL method to calculate default warp target properties
            transform, width, height = calculate_default_transform(
                tile.tp.crs,
                dst_pyramid.crs,
                tile.width,
                tile.height,
                *tile.bounds
            )
            # this is the resolution the tile would have in destination TilePyramid CRS
            tile_resolution = round(transform[0], precision)
        elif matching_method == "min":
            # calculate the minimum pixel size from the four tile corner pixels
            l, b, r, t = tile.bounds
            x = tile.pixel_x_size
            y = tile.pixel_y_size
            res = []
            for bounds in [
                (l, t - y, l + x, t),  # left top
                (l, b, l + x, b + y),  # left bottom
                (r - x, b, r, b + y),  # right bottom
                (r - x, t - y, r, t)   # right top
            ]:
                try:
                    w, h = width_height(bounds)
                    res.extend([w, h])
                except TopologicalError:
                    logger.debug("pixel outside of destination pyramid")
            if res:
                tile_resolution = round(min(res), precision)
            else:
                raise TopologicalError("tile outside of destination pyramid")
        else:
            raise ValueError("invalid method given: %s", matching_method)
        logger.debug(
            "we are looking for a zoom level interpolating to %s resolution",
            tile_resolution
        )
        zoom = 0
        while True:
            td_resolution = round(dst_pyramid.pixel_x_size(zoom), precision)
            if td_resolution <= tile_resolution:
                break
            zoom += 1
        logger.debug("target zoom for %s: %s (%s)", tile_resolution, zoom, td_resolution)
        return zoom

Determine the best zoom level in target TilePyramid from given Tile .


def path_is_remote(path, s3=True):
    """
    Determine whether file path is remote or local.

    Parameters
    ----------
    path : path to file

    Returns
    -------
    is_remote : bool
    """
    prefixes = ("http://", "https://", "/vsicurl/")
    if s3:
        prefixes += ("s3://", "/vsis3/")
    return path.startswith(prefixes)

def path_is_remote ( path , s3 = True ) : prefixes = ( "http://" , "https://" , "/vsicurl/" ) if s3 : prefixes += ( "s3://" , "/vsis3/" ) return path . startswith ( prefixes )

Determine whether file path is remote or local .


def path_exists(path):
    """
    Check if file exists either remote or local.

    Parameters:
    -----------
    path : path to file

    Returns:
    --------
    exists : bool
    """
    if path.startswith(("http://", "https://")):
        try:
            urlopen(path).info()
            return True
        except HTTPError as e:
            if e.code == 404:
                return False
            else:
                raise
    elif path.startswith("s3://"):
        bucket = get_boto3_bucket(path.split("/")[2])
        key = "/".join(path.split("/")[3:])
        for obj in bucket.objects.filter(Prefix=key):
            if obj.key == key:
                return True
        else:
            return False
    else:
        logger.debug("%s exists: %s", path, os.path.exists(path))
        return os.path.exists(path)

def
path_exists
(
path
)
:
if
path
.
startswith
(
(
"http://"
,
"https://"
)
)
:
try
:
urlopen
(
path
)
.
info
(
)
return
True
except
HTTPError
as
e
:
if
e
.
code
==
404
:
return
False
else
:
raise
elif
path
.
startswith
(
"s3://"
)
:
bucket
=
get_boto3_bucket
(
path
.
split
(
"/"
)
[
2
]
)
key
=
"/"
.
join
(
path
.
split
(
"/"
)
[
3
:
]
)
for
obj
in
bucket
.
objects
.
filter
(
Prefix
=
key
)
:
if
obj
.
key
==
key
:
return
True
else
:
return
False
else
:
logger
.
debug
(
"%s exists: %s"
,
path
,
os
.
path
.
exists
(
path
)
)
return
os
.
path
.
exists
(
path
)

Check if file exists either remote or local .


def absolute_path(path=None, base_dir=None):
    """
    Return absolute path if path is local.

    Parameters:
    -----------
    path : path to file
    base_dir : base directory used for absolute path

    Returns:
    --------
    absolute path
    """
    if path_is_remote(path):
        return path
    else:
        if os.path.isabs(path):
            return path
        else:
            if base_dir is None or not os.path.isabs(base_dir):
                raise TypeError("base_dir must be an absolute path.")
            return os.path.abspath(os.path.join(base_dir, path))

def
absolute_path
(
path
=
,
base_dir
=
)
:
if
path_is_remote
(
path
)
:
return
path
else
:
if
os
.
path
.
isabs
(
path
)
:
return
path
else
:
if
base_dir
is
or
not
os
.
path
.
isabs
(
base_dir
)
:
raise
TypeError
(
"base_dir must be an absolute path."
)
return
os
.
path
.
abspath
(
os
.
path
.
join
(
base_dir
,
path
)
)


def relative_path(path=None, base_dir=None):
    """
    Return relative path if path is local.

    Parameters:
    -----------
    path : path to file
    base_dir : directory where path sould be relative to

    Returns:
    --------
    relative path
    """
    if path_is_remote(path) or not os.path.isabs(path):
        return path
    else:
        return os.path.relpath(path, base_dir)

def relative_path ( path = None , base_dir = None ) : if path_is_remote ( path ) or not os . path . isabs ( path ) : return path else : return os . path . relpath ( path , base_dir )


def write_json(path, params):
    """Write local or remote."""
    logger.debug("write %s to %s", params, path)
    if path.startswith("s3://"):
        bucket = get_boto3_bucket(path.split("/")[2])
        key = "/".join(path.split("/")[3:])
        logger.debug("upload %s", key)
        bucket.put_object(
            Key=key,
            Body=json.dumps(params, sort_keys=True, indent=4)
        )
    else:
        makedirs(os.path.dirname(path))
        with open(path, 'w') as dst:
            json.dump(params, dst, sort_keys=True, indent=4)

def
write_json
(
path
,
params
)
:
logger
.
debug
(
"write %s to %s"
,
params
,
path
)
if
path
.
startswith
(
"s3://"
)
:
bucket
=
get_boto3_bucket
(
path
.
split
(
"/"
)
[
2
]
)
key
=
"/"
.
join
(
path
.
split
(
"/"
)
[
3
:
]
)
logger
.
debug
(
"upload %s"
,
key
)
bucket
.
put_object
(
Key
=
key
,
Body
=
json
.
dumps
(
params
,
sort_keys
=
True
,
indent
=
4
)
)
else
:
makedirs
(
os
.
path
.
dirname
(
path
)
)
with
open
(
path
,
'w'
)
as
dst
:
json
.
dump
(
params
,
dst
,
sort_keys
=
True
,
indent
=
4
)


def read_json(path):
    """Read local or remote."""
    if path.startswith(("http://", "https://")):
        try:
            return json.loads(urlopen(path).read().decode())
        except HTTPError:
            raise FileNotFoundError("%s not found", path)
    elif path.startswith("s3://"):
        bucket = get_boto3_bucket(path.split("/")[2])
        key = "/".join(path.split("/")[3:])
        for obj in bucket.objects.filter(Prefix=key):
            if obj.key == key:
                return json.loads(obj.get()['Body'].read().decode())
        raise FileNotFoundError("%s not found", path)
    else:
        try:
            with open(path, "r") as src:
                return json.loads(src.read())
        except:
            raise FileNotFoundError("%s not found", path)

def
read_json
(
path
)
:
if
path
.
startswith
(
(
"http://"
,
"https://"
)
)
:
try
:
return
json
.
loads
(
urlopen
(
path
)
.
read
(
)
.
decode
(
)
)
except
HTTPError
:
raise
FileNotFoundError
(
"%s not found"
,
path
)
elif
path
.
startswith
(
"s3://"
)
:
bucket
=
get_boto3_bucket
(
path
.
split
(
"/"
)
[
2
]
)
key
=
"/"
.
join
(
path
.
split
(
"/"
)
[
3
:
]
)
for
obj
in
bucket
.
objects
.
filter
(
Prefix
=
key
)
:
if
obj
.
key
==
key
:
return
json
.
loads
(
obj
.
get
(
)
[
'Body'
]
.
read
(
)
.
decode
(
)
)
raise
FileNotFoundError
(
"%s not found"
,
path
)
else
:
try
:
with
open
(
path
,
"r"
)
as
src
:
return
json
.
loads
(
src
.
read
(
)
)
except
:
raise
FileNotFoundError
(
"%s not found"
,
path
)


def hook(self, event_type='push'):
        """
        Registers a function as a hook. Multiple hooks can be registered for a given type, but the
        order in which they are invoke is unspecified.

        :param event_type: The event type this hook will be invoked for.
        """

        def decorator(func):
            self._hooks[event_type].append(func)
            return func

        return decorator

def hook ( self , event_type = 'push' ) : def decorator ( func ) : self . _hooks [ event_type ] . append ( func ) return func return decorator

Registers a function as a hook . Multiple hooks can be registered for a given type but the order in which they are invoke is unspecified .


def _get_digest(self):
        """Return message digest if a secret key was provided"""

        return hmac.new(
            self._secret, request.data, hashlib.sha1).hexdigest() if self._secret else None

def _get_digest ( self ) : return hmac . new ( self . _secret , request . data , hashlib . sha1 ) . hexdigest ( ) if self . _secret else None


def _postreceive(self):
        """Callback from Flask"""

        digest = self._get_digest()

        if digest is not None:
            sig_parts = _get_header('X-Hub-Signature').split('=', 1)
            if not isinstance(digest, six.text_type):
                digest = six.text_type(digest)

            if (len(sig_parts) < 2 or sig_parts[0] != 'sha1'
                    or not hmac.compare_digest(sig_parts[1], digest)):
                abort(400, 'Invalid signature')

        event_type = _get_header('X-Github-Event')
        data = request.get_json()

        if data is None:
            abort(400, 'Request body must contain json')

        self._logger.info(
            '%s (%s)', _format_event(event_type, data), _get_header('X-Github-Delivery'))

        for hook in self._hooks.get(event_type, []):
            hook(data)

        return '', 204

def
_postreceive
(
self
)
:
digest
=
self
.
_get_digest
(
)
if
digest
is
not
:
sig_parts
=
_get_header
(
'X-Hub-Signature'
)
.
split
(
'='
,
1
)
if
not
isinstance
(
digest
,
six
.
text_type
)
:
digest
=
six
.
text_type
(
digest
)
if
(
len
(
sig_parts
)
<
2
or
sig_parts
[
0
]
!=
'sha1'
or
not
hmac
.
compare_digest
(
sig_parts
[
1
]
,
digest
)
)
:
abort
(
400
,
'Invalid signature'
)
event_type
=
_get_header
(
'X-Github-Event'
)
data
=
request
.
get_json
(
)
if
data
is
:
abort
(
400
,
'Request body must contain json'
)
self
.
_logger
.
info
(
'%s (%s)'
,
_format_event
(
event_type
,
data
)
,
_get_header
(
'X-Github-Delivery'
)
)
for
hook
in
self
.
_hooks
.
get
(
event_type
,
[
]
)
:
hook
(
data
)
return
''
,
204


def long_description():
    """Generate .rst document for PyPi."""
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--doc', dest="doc",
            action="store_true", default=False)
    args, sys.argv = parser.parse_known_args(sys.argv)
    if args.doc:
        import doc2md, pypandoc
        md = doc2md.doc2md(doc2md.__doc__, "doc2md", toc=False)
        long_description = pypandoc.convert(md, 'rst', format='md')
    else:
        return None

def long_description ( ) : import argparse parser = argparse . ArgumentParser ( ) parser . add_argument ( '--doc' , dest = "doc" , action = "store_true" , default = False ) args , sys . argv = parser . parse_known_args ( sys . argv ) if args . doc : import doc2md , pypandoc md = doc2md . doc2md ( doc2md . __doc__ , "doc2md" , toc = False ) long_description = pypandoc . convert ( md , 'rst' , format = 'md' ) else : return None


def unindent(lines):
    """
    Remove common indentation from string.

    Unlike doctrim there is no special treatment of the first line.

    """
    try:
        # Determine minimum indentation:
        indent = min(len(line) - len(line.lstrip())
                     for line in lines if line)
    except ValueError:
        return lines
    else:
        return [line[indent:] for line in lines]

def
unindent
(
lines
)
:
try
:
# Determine minimum indentation:
indent
=
min
(
len
(
line
)
len
(
line
.
lstrip
(
)
)
for
line
in
lines
if
line
)
except
ValueError
:
return
lines
else
:
return
[
line
[
indent
:
]
for
line
in
lines
]


def find_sections(lines):
    """
    Find all section names and return a list with their names.
    """
    sections = []
    for line in lines:
        if is_heading(line):
            sections.append(get_heading(line))
    return sections

def find_sections ( lines ) : sections = [ ] for line in lines : if is_heading ( line ) : sections . append ( get_heading ( line ) ) return sections

Find all section names and return a list with their names .


def make_toc(sections, maxdepth=0):
    """
    Generate table of contents for array of section names.
    """
    if not sections:
        return []
    outer = min(n for n,t in sections)
    refs = []
    for ind,sec in sections:
        if maxdepth and ind-outer+1 > maxdepth:
            continue
        ref = sec.lower()
        ref = ref.replace('`', '')
        ref = ref.replace(' ', '-')
        ref = ref.replace('?', '')
        refs.append("    "*(ind-outer) + "- [%s](#%s)" % (sec, ref))
    return refs

def
make_toc
(
sections
,
maxdepth
=
0
)
:
if
not
sections
:
return
[
]
outer
=
min
(
n
for
n
,
t
in
sections
)
refs
=
[
]
for
ind
,
sec
in
sections
:
if
maxdepth
and
ind
outer
+
1
>
maxdepth
:
continue
ref
=
sec
.
lower
(
)
ref
=
ref
.
replace
(
'`'
,
''
)
ref
=
ref
.
replace
(
' '
,
'-'
)
ref
=
ref
.
replace
(
'?'
,
''
)
refs
.
append
(
"    "
*
(
ind
outer
)
+
"- [%s](#%s)"
%
(
sec
,
ref
)
)
return
refs

Generate table of contents for array of section names .


def doc2md(docstr, title, min_level=1, more_info=False, toc=True, maxdepth=0):
    """
    Convert a docstring to a markdown text.
    """
    text = doctrim(docstr)
    lines = text.split('\n')

    sections = find_sections(lines)
    if sections:
        level = min(n for n,t in sections) - 1
    else:
        level = 1

    shiftlevel = 0
    if level < min_level:
        shiftlevel = min_level - level
        level = min_level
        sections = [(lev+shiftlevel, tit) for lev,tit in sections]

    head = next((i for i, l in enumerate(lines) if is_heading(l)), 0)
    md = [
        make_heading(level, title),
        "",
    ] + lines[:head]
    if toc:
        md += make_toc(sections, maxdepth)
        md += ['']
    md += _doc2md(lines[head:], shiftlevel)
    if more_info:
        return (md, sections)
    else:
        return "\n".join(md)

def doc2md ( docstr , title , min_level = 1 , more_info = False , toc = True , maxdepth = 0 ) : text = doctrim ( docstr ) lines = text . split ( '\n' ) sections = find_sections ( lines ) if sections : level = min ( n for n , t in sections ) - 1 else : level = 1 shiftlevel = 0 if level < min_level : shiftlevel = min_level - level level = min_level sections = [ ( lev + shiftlevel , tit ) for lev , tit in sections ] head = next ( ( i for i , l in enumerate ( lines ) if is_heading ( l ) ) , 0 ) md = [ make_heading ( level , title ) , "" , ] + lines [ : head ] if toc : md += make_toc ( sections , maxdepth ) md += [ '' ] md += _doc2md ( lines [ head : ] , shiftlevel ) if more_info : return ( md , sections ) else : return "\n" . join ( md )

Convert a docstring to a markdown text .


def mod2md(module, title, title_api_section, toc=True, maxdepth=0):
    """
    Generate markdown document from module, including API section.
    """
    docstr = module.__doc__

    text = doctrim(docstr)
    lines = text.split('\n')

    sections = find_sections(lines)
    if sections:
        level = min(n for n,t in sections) - 1
    else:
        level = 1

    api_md = []
    api_sec = []
    if title_api_section and module.__all__:
        sections.append((level+1, title_api_section))
        for name in module.__all__:
            api_sec.append((level+2, "`" + name + "`"))
            api_md += ['', '']
            entry = module.__dict__[name]
            if entry.__doc__:
                md, sec = doc2md(entry.__doc__, "`" + name + "`",
                        min_level=level+2, more_info=True, toc=False)
                api_sec += sec
                api_md += md

    sections += api_sec

    # headline
    head = next((i for i, l in enumerate(lines) if is_heading(l)), 0)
    md = [
        make_heading(level, title),
        "",
    ] + lines[:head]

    # main sections
    if toc:
        md += make_toc(sections, maxdepth)
        md += ['']
    md += _doc2md(lines[head:])

    # API section
    md += [
        '',
        '',
        make_heading(level+1, title_api_section),
    ]
    if toc:
        md += ['']
        md += make_toc(api_sec, 1)
    md += api_md

    return "\n".join(md)

def
mod2md
(
module
,
title
,
title_api_section
,
toc
=
True
,
maxdepth
=
0
)
:
docstr
=
module
.
__doc__
text
=
doctrim
(
docstr
)
lines
=
text
.
split
(
'\n'
)
sections
=
find_sections
(
lines
)
if
sections
:
level
=
min
(
n
for
n
,
t
in
sections
)
1
else
:
level
=
1
api_md
=
[
]
api_sec
=
[
]
if
title_api_section
and
module
.
__all__
:
sections
.
append
(
(
level
+
1
,
title_api_section
)
)
for
name
in
module
.
__all__
:
api_sec
.
append
(
(
level
+
2
,
"`"
+
name
+
"`"
)
)
api_md
+=
[
''
,
''
]
entry
=
module
.
__dict__
[
name
]
if
entry
.
__doc__
:
md
,
sec
=
doc2md
(
entry
.
__doc__
,
"`"
+
name
+
"`"
,
min_level
=
level
+
2
,
more_info
=
True
,
toc
=
False
)
api_sec
+=
sec
api_md
+=
md
sections
+=
api_sec
# headline
head
=
next
(
(
i
for
i
,
l
in
enumerate
(
lines
)
if
is_heading
(
l
)
)
,
0
)
md
=
[
make_heading
(
level
,
title
)
,
""
,
]
+
lines
[
:
head
]
# main sections
if
toc
:
md
+=
make_toc
(
sections
,
maxdepth
)
md
+=
[
''
]
md
+=
_doc2md
(
lines
[
head
:
]
)
# API section
md
+=
[
''
,
''
,
make_heading
(
level
+
1
,
title_api_section
)
,
]
if
toc
:
md
+=
[
''
]
md
+=
make_toc
(
api_sec
,
1
)
md
+=
api_md
return
"\n"
.
join
(
md
)

Generate markdown document from module including API section .


def largest_finite_distance(self):
        """
        Compute the maximum temporal distance.

        Returns
        -------
        max_temporal_distance : float
        """
        block_start_distances = [block.distance_start for block in self._profile_blocks if
                                 block.distance_start < float('inf')]
        block_end_distances = [block.distance_end for block in self._profile_blocks if
                               block.distance_end < float('inf')]
        distances = block_start_distances + block_end_distances
        if len(distances) > 0:
            return max(distances)
        else:
            return None

def largest_finite_distance ( self ) : block_start_distances = [ block . distance_start for block in self . _profile_blocks if block . distance_start < float ( 'inf' ) ] block_end_distances = [ block . distance_end for block in self . _profile_blocks if block . distance_end < float ( 'inf' ) ] distances = block_start_distances + block_end_distances if len ( distances ) > 0 : return max ( distances ) else : return None


def _temporal_distance_cdf(self):
        """
        Temporal distance cumulative density function.

        Returns
        -------
        x_values: numpy.array
            values for the x-axis
        cdf: numpy.array
            cdf values
        """
        distance_split_points = set()
        for block in self._profile_blocks:
            if block.distance_start != float('inf'):
                distance_split_points.add(block.distance_end)
                distance_split_points.add(block.distance_start)

        distance_split_points_ordered = numpy.array(sorted(list(distance_split_points)))
        temporal_distance_split_widths = distance_split_points_ordered[1:] - distance_split_points_ordered[:-1]
        trip_counts = numpy.zeros(len(temporal_distance_split_widths))
        delta_peaks = defaultdict(lambda: 0)

        for block in self._profile_blocks:
            if block.distance_start == block.distance_end:
                delta_peaks[block.distance_end] += block.width()
            else:
                start_index = numpy.searchsorted(distance_split_points_ordered, block.distance_end)
                end_index = numpy.searchsorted(distance_split_points_ordered, block.distance_start)
                trip_counts[start_index:end_index] += 1

        unnormalized_cdf = numpy.array([0] + list(numpy.cumsum(temporal_distance_split_widths * trip_counts)))
        if not (numpy.isclose(
                [unnormalized_cdf[-1]],
                [self._end_time - self._start_time - sum(delta_peaks.values())], atol=1E-4
        ).all()):
            print(unnormalized_cdf[-1], self._end_time - self._start_time - sum(delta_peaks.values()))
            raise RuntimeError("Something went wrong with cdf computation!")

        if len(delta_peaks) > 0:
            for peak in delta_peaks.keys():
                if peak == float('inf'):
                    continue
                index = numpy.nonzero(distance_split_points_ordered == peak)[0][0]
                unnormalized_cdf = numpy.insert(unnormalized_cdf, index, unnormalized_cdf[index])
                distance_split_points_ordered = numpy.insert(distance_split_points_ordered, index,
                                                             distance_split_points_ordered[index])
                # walk_waiting_time_fraction = walk_total_time / (self.end_time_dep - self.start_time_dep)
                unnormalized_cdf[(index + 1):] = unnormalized_cdf[(index + 1):] + delta_peaks[peak]

        norm_cdf = unnormalized_cdf / (unnormalized_cdf[-1] + delta_peaks[float('inf')])
        return distance_split_points_ordered, norm_cdf

Temporal distance cumulative density function .


def _temporal_distance_pdf(self):
        """
        Temporal distance probability density function.

        Returns
        -------
        non_delta_peak_split_points: numpy.array
        non_delta_peak_densities: numpy.array
            len(density) == len(temporal_distance_split_points_ordered) -1
        delta_peak_loc_to_probability_mass : dict
        """
        temporal_distance_split_points_ordered, norm_cdf = self._temporal_distance_cdf()
        delta_peak_loc_to_probability_mass = {}

        non_delta_peak_split_points = [temporal_distance_split_points_ordered[0]]
        non_delta_peak_densities = []
        for i in range(0, len(temporal_distance_split_points_ordered) - 1):
            left = temporal_distance_split_points_ordered[i]
            right = temporal_distance_split_points_ordered[i + 1]
            width = right - left
            prob_mass = norm_cdf[i + 1] - norm_cdf[i]
            if width == 0.0:
                delta_peak_loc_to_probability_mass[left] = prob_mass
            else:
                non_delta_peak_split_points.append(right)
                non_delta_peak_densities.append(prob_mass / float(width))
        assert (len(non_delta_peak_densities) == len(non_delta_peak_split_points) - 1)
        return numpy.array(non_delta_peak_split_points), \
               numpy.array(non_delta_peak_densities), delta_peak_loc_to_probability_mass

def _temporal_distance_pdf ( self ) : temporal_distance_split_points_ordered , norm_cdf = self . _temporal_distance_cdf ( ) delta_peak_loc_to_probability_mass = { } non_delta_peak_split_points = [ temporal_distance_split_points_ordered [ 0 ] ] non_delta_peak_densities = [ ] for i in range ( 0 , len ( temporal_distance_split_points_ordered ) - 1 ) : left = temporal_distance_split_points_ordered [ i ] right = temporal_distance_split_points_ordered [ i + 1 ] width = right - left prob_mass = norm_cdf [ i + 1 ] - norm_cdf [ i ] if width == 0.0 : delta_peak_loc_to_probability_mass [ left ] = prob_mass else : non_delta_peak_split_points . append ( right ) non_delta_peak_densities . append ( prob_mass / float ( width ) ) assert ( len ( non_delta_peak_densities ) == len ( non_delta_peak_split_points ) - 1 ) return numpy . array ( non_delta_peak_split_points ) , numpy . array ( non_delta_peak_densities ) , delta_peak_loc_to_probability_mass

Temporal distance probability density function .


def remove_all_trips_fully_outside_buffer(db_conn, center_lat, center_lon, buffer_km, update_secondary_data=True):
    """
    Not used in the regular filter process for the time being.

    Parameters
    ----------
    db_conn: sqlite3.Connection
        connection to the GTFS object
    center_lat: float
    center_lon: float
    buffer_km: float
    """
    distance_function_str = add_wgs84_distance_function_to_db(db_conn)
    stops_within_buffer_query_sql = "SELECT stop_I FROM stops WHERE CAST(" + distance_function_str + \
                                "(lat, lon, {lat} , {lon}) AS INT) < {d_m}"\
        .format(lat=float(center_lat), lon=float(center_lon), d_m=int(1000*buffer_km))
    select_all_trip_Is_where_stop_I_is_within_buffer_sql = "SELECT distinct(trip_I) FROM stop_times WHERE stop_I IN (" + stops_within_buffer_query_sql + ")"
    trip_Is_to_remove_sql = "SELECT trip_I FROM trips WHERE trip_I NOT IN ( " + select_all_trip_Is_where_stop_I_is_within_buffer_sql + ")"
    trip_Is_to_remove = pandas.read_sql(trip_Is_to_remove_sql, db_conn)["trip_I"].values
    trip_Is_to_remove_string = ",".join([str(trip_I) for trip_I in trip_Is_to_remove])
    remove_all_trips_fully_outside_buffer_sql = "DELETE FROM trips WHERE trip_I IN (" + trip_Is_to_remove_string + ")"
    remove_all_stop_times_where_trip_I_fully_outside_buffer_sql = "DELETE FROM stop_times WHERE trip_I IN (" + trip_Is_to_remove_string  + ")"
    db_conn.execute(remove_all_trips_fully_outside_buffer_sql)
    db_conn.execute(remove_all_stop_times_where_trip_I_fully_outside_buffer_sql)
    delete_stops_not_in_stop_times_and_not_as_parent_stop(db_conn)
    db_conn.execute(DELETE_ROUTES_NOT_PRESENT_IN_TRIPS_SQL)
    db_conn.execute(DELETE_SHAPES_NOT_REFERENCED_IN_TRIPS_SQL)
    db_conn.execute(DELETE_DAYS_ENTRIES_NOT_PRESENT_IN_TRIPS_SQL)
    db_conn.execute(DELETE_DAY_TRIPS2_ENTRIES_NOT_PRESENT_IN_TRIPS_SQL)
    db_conn.execute(DELETE_CALENDAR_ENTRIES_FOR_NON_REFERENCE_SERVICE_IS_SQL)
    db_conn.execute(DELETE_CALENDAR_DATES_ENTRIES_FOR_NON_REFERENCE_SERVICE_IS_SQL)
    db_conn.execute(DELETE_FREQUENCIES_ENTRIES_NOT_PRESENT_IN_TRIPS)
    db_conn.execute(DELETE_AGENCIES_NOT_REFERENCED_IN_ROUTES_SQL)
    if update_secondary_data:
        update_secondary_data_copies(db_conn)

Not used in the regular filter process for the time being .


def remove_dangling_shapes(db_conn):
    """
    Remove dangling entries from the shapes directory.

    Parameters
    ----------
    db_conn: sqlite3.Connection
        connection to the GTFS object
    """
    db_conn.execute(DELETE_SHAPES_NOT_REFERENCED_IN_TRIPS_SQL)
    SELECT_MIN_MAX_SHAPE_BREAKS_BY_TRIP_I_SQL = \
        "SELECT trips.trip_I, shape_id, min(shape_break) as min_shape_break, max(shape_break) as max_shape_break FROM trips, stop_times WHERE trips.trip_I=stop_times.trip_I GROUP BY trips.trip_I"
    trip_min_max_shape_seqs= pandas.read_sql(SELECT_MIN_MAX_SHAPE_BREAKS_BY_TRIP_I_SQL, db_conn)

    rows = []
    for row in trip_min_max_shape_seqs.itertuples():
        shape_id, min_shape_break, max_shape_break = row.shape_id, row.min_shape_break, row.max_shape_break
        if min_shape_break is None or max_shape_break is None:
            min_shape_break = float('-inf')
            max_shape_break = float('-inf')
        rows.append( (shape_id, min_shape_break, max_shape_break) )
    DELETE_SQL_BASE = "DELETE FROM shapes WHERE shape_id=? AND (seq<? OR seq>?)"
    db_conn.executemany(DELETE_SQL_BASE, rows)
    remove_dangling_shapes_references(db_conn)

Remove dangling entries from the shapes directory .


def compute_pseudo_connections(transit_connections, start_time_dep,
                               end_time_dep, transfer_margin,
                               walk_network, walk_speed):
    """
    Given a set of transit events and the static walk network,
    "transform" the static walking network into a set of "pseudo-connections".

    As a first approximation, we add pseudo-connections to depart after each arrival of a transit connection
    to it's arrival stop.

    Parameters
    ----------
    transit_connections: list[Connection]
    start_time_dep : int
        start time in unixtime seconds
    end_time_dep: int
        end time in unixtime seconds (no new connections will be scanned after this time)
    transfer_margin: int
        required extra margin required for transfers in seconds
    walk_speed: float
        walking speed between stops in meters / second
    walk_network: networkx.Graph
        each edge should have the walking distance as a data attribute ("d_walk") expressed in meters

    Returns
    -------
    pseudo_connections: set[Connection]
    """
    # A pseudo-connection should be created after (each) arrival to a transit_connection's arrival stop.
    pseudo_connection_set = set()  # use a set to ignore possible duplicates
    for c in transit_connections:
        if start_time_dep <= c.departure_time <= end_time_dep:
            walk_arr_stop = c.departure_stop
            walk_arr_time = c.departure_time - transfer_margin
            for _, walk_dep_stop, data in walk_network.edges(nbunch=[walk_arr_stop], data=True):
                walk_dep_time = walk_arr_time - data['d_walk'] / float(walk_speed)
                if walk_dep_time > end_time_dep or walk_dep_time < start_time_dep:
                    continue
                pseudo_connection = Connection(walk_dep_stop,
                                               walk_arr_stop,
                                               walk_dep_time,
                                               walk_arr_time,
                                               Connection.WALK_TRIP_ID,
                                               Connection.WALK_SEQ,
                                               is_walk=True)
                pseudo_connection_set.add(pseudo_connection)
    return pseudo_connection_set

Given a set of transit events and the static walk network transform the static walking network into a set of pseudo - connections .


def get_min_visit_time(self):
        """
        Get the earliest visit time of the stop.
        """
        if not self.visit_events:
            return float('inf')
        else:
            return min(self.visit_events, key=lambda event: event.arr_time_ut).arr_time_ut

def get_min_visit_time ( self ) : if not self . visit_events : return float ( 'inf' ) else : return min ( self . visit_events , key = lambda event : event . arr_time_ut ) . arr_time_ut

Get the earliest visit time of the stop .


def can_infect(self, event):
        """
        Whether the spreading stop can infect using this event.
        """
        if event.from_stop_I != self.stop_I:
            return False

        if not self.has_been_visited():
            return False
        else:
            time_sep = event.dep_time_ut-self.get_min_visit_time()
            # if the gap between the earliest visit_time and current time is
            # smaller than the min. transfer time, the stop can pass the spreading
            # forward
            if (time_sep >= self.min_transfer_time) or (event.trip_I == -1 and time_sep >= 0):
                return True
            else:
                for visit in self.visit_events:
                    # if no transfer, please hop-on
                    if (event.trip_I == visit.trip_I) and (time_sep >= 0):
                        return True
            return False

Whether the spreading stop can infect using this event .


def make_views(cls, conn):
        """Create day_trips and day_stop_times views.

        day_trips:  day_trips2 x trips  = days x trips
        day_stop_times: day_trips2 x trips x stop_times = days x trips x stop_times
        """
        conn.execute('DROP VIEW IF EXISTS main.day_trips')
        conn.execute('CREATE VIEW day_trips AS   '
                     'SELECT day_trips2.*, trips.* '
                     #'days.day_start_ut+trips.start_time_ds AS start_time_ut, '
                     #'days.day_start_ut+trips.end_time_ds AS end_time_ut   '
                     'FROM day_trips2 JOIN trips USING (trip_I);')
        conn.commit()

        conn.execute('DROP VIEW IF EXISTS main.day_stop_times')
        conn.execute('CREATE VIEW day_stop_times AS   '
                     'SELECT day_trips2.*, trips.*, stop_times.*, '
                     #'days.day_start_ut+trips.start_time_ds AS start_time_ut, '
                     #'days.day_start_ut+trips.end_time_ds AS end_time_ut, '
                     'day_trips2.day_start_ut+stop_times.arr_time_ds AS arr_time_ut, '
                     'day_trips2.day_start_ut+stop_times.dep_time_ds AS dep_time_ut   '
                     'FROM day_trips2 '
                     'JOIN trips USING (trip_I) '
                     'JOIN stop_times USING (trip_I)')
        conn.commit()

Create day_trips and day_stop_times views .


def createcolorbar(cmap, norm):
    """Create a colourbar with limits of lwr and upr"""
    cax, kw = matplotlib.colorbar.make_axes(matplotlib.pyplot.gca())
    c = matplotlib.colorbar.ColorbarBase(cax, cmap=cmap, norm=norm)
    return c

def createcolorbar ( cmap , norm ) : cax , kw = matplotlib . colorbar . make_axes ( matplotlib . pyplot . gca ( ) ) c = matplotlib . colorbar . ColorbarBase ( cax , cmap = cmap , norm = norm ) return c


def write_temporal_networks_by_route_type(gtfs, extract_output_dir):
    """
    Write temporal networks by route type to disk.

    Parameters
    ----------
    gtfs: gtfspy.GTFS
    extract_output_dir: str
    """
    util.makedirs(extract_output_dir)
    for route_type in route_types.TRANSIT_ROUTE_TYPES:
        pandas_data_frame = temporal_network(gtfs, start_time_ut=None, end_time_ut=None, route_type=route_type)
        tag = route_types.ROUTE_TYPE_TO_LOWERCASE_TAG[route_type]
        out_file_name = os.path.join(extract_output_dir, tag + ".tnet")
        pandas_data_frame.to_csv(out_file_name, encoding='utf-8', index=False)

def write_temporal_networks_by_route_type ( gtfs , extract_output_dir ) : util . makedirs ( extract_output_dir ) for route_type in route_types . TRANSIT_ROUTE_TYPES : pandas_data_frame = temporal_network ( gtfs , start_time_ut = None , end_time_ut = None , route_type = route_type ) tag = route_types . ROUTE_TYPE_TO_LOWERCASE_TAG [ route_type ] out_file_name = os . path . join ( extract_output_dir , tag + ".tnet" ) pandas_data_frame . to_csv ( out_file_name , encoding = 'utf-8' , index = False )

Write temporal networks by route type to disk .


def _write_stop_to_stop_network_edges(net, file_name, data=True, fmt=None):
    """
    Write out a network

    Parameters
    ----------
    net: networkx.DiGraph
    base_name: str
        path to the filename (without extension)
    data: bool, optional
        whether or not to write out any edge data present
    fmt: str, optional
        If "csv" write out the network in csv format.
    """
    if fmt is None:
        fmt = "edg"

    if fmt == "edg":
        if data:
            networkx.write_edgelist(net, file_name, data=True)
        else:
            networkx.write_edgelist(net, file_name)
    elif fmt == "csv":
        with open(file_name, 'w') as f:
            # writing out the header
            edge_iter = net.edges_iter(data=True)
            _, _, edg_data = next(edge_iter)
            edg_data_keys = list(sorted(edg_data.keys()))
            header = ";".join(["from_stop_I", "to_stop_I"] + edg_data_keys)
            f.write(header)
            for from_node_I, to_node_I, data in net.edges_iter(data=True):
                f.write("\n")
                values = [str(from_node_I), str(to_node_I)]
                data_values = []
                for key in edg_data_keys:
                    if key == "route_I_counts":
                        route_I_counts_string = str(data[key]).replace(" ", "")[1:-1]
                        data_values.append(route_I_counts_string)
                    else:
                        data_values.append(str(data[key]))
                all_values = values + data_values
                f.write(";".join(all_values))

def
_write_stop_to_stop_network_edges
(
net
,
file_name
,
data
=
True
,
fmt
=
)
:
if
fmt
is
:
fmt
=
"edg"
if
fmt
==
"edg"
:
if
data
:
networkx
.
write_edgelist
(
net
,
file_name
,
data
=
True
)
else
:
networkx
.
write_edgelist
(
net
,
file_name
)
elif
fmt
==
"csv"
:
with
open
(
file_name
,
'w'
)
as
f
:
# writing out the header
edge_iter
=
net
.
edges_iter
(
data
=
True
)
_
,
_
,
edg_data
=
next
(
edge_iter
)
edg_data_keys
=
list
(
sorted
(
edg_data
.
keys
(
)
)
)
header
=
";"
.
join
(
[
"from_stop_I"
,
"to_stop_I"
]
+
edg_data_keys
)
f
.
write
(
header
)
for
from_node_I
,
to_node_I
,
data
in
net
.
edges_iter
(
data
=
True
)
:
f
.
write
(
"\n"
)
values
=
[
str
(
from_node_I
)
,
str
(
to_node_I
)
]
data_values
=
[
]
for
key
in
edg_data_keys
:
if
key
==
"route_I_counts"
:
route_I_counts_string
=
str
(
data
[
key
]
)
.
replace
(
" "
,
""
)
[
1
:
1
]
data_values
.
append
(
route_I_counts_string
)
else
:
data_values
.
append
(
str
(
data
[
key
]
)
)
all_values
=
values
+
data_values
f
.
write
(
";"
.
join
(
all_values
)
)


def write_gtfs(gtfs, output):
    """
    Write out the database according to the GTFS format.

    Parameters
    ----------
    gtfs: gtfspy.GTFS
    output: str
        Path where to put the GTFS files
        if output ends with ".zip" a ZIP-file is created instead.

    Returns
    -------
    None
    """
    output = os.path.abspath(output)
    uuid_str = "tmp_" + str(uuid.uuid1())
    if output[-4:] == '.zip':
        zip = True
        out_basepath = os.path.dirname(os.path.abspath(output))
        if not os.path.exists(out_basepath):
            raise IOError(out_basepath + " does not exist, cannot write gtfs as a zip")
        tmp_dir = os.path.join(out_basepath, str(uuid_str))
        # zip_file_na,e = ../out_basedir + ".zip
    else:
        zip = False
        out_basepath = output
        tmp_dir = os.path.join(out_basepath + "_" + str(uuid_str))

    os.makedirs(tmp_dir, exist_ok=True)

    gtfs_table_to_writer = {
        "agency": _write_gtfs_agencies,
        "calendar": _write_gtfs_calendar,
        "calendar_dates": _write_gtfs_calendar_dates,
        # fare attributes and fare_rules omitted (seldomly used)
        "feed_info": _write_gtfs_feed_info,
        # "frequencies": not written, as they are incorporated into trips and routes,
        # Frequencies table is expanded into other tables on initial import. -> Thus frequencies.txt is not created
        "routes": _write_gtfs_routes,
        "shapes": _write_gtfs_shapes,
        "stops": _write_gtfs_stops,
        "stop_times": _write_gtfs_stop_times,
        "transfers": _write_gtfs_transfers,
        "trips": _write_gtfs_trips,
    }

    for table, writer in gtfs_table_to_writer.items():
        fname_to_write = os.path.join(tmp_dir, table + '.txt')
        print(fname_to_write)
        writer(gtfs, open(os.path.join(tmp_dir, table + '.txt'), 'w'))

    if zip:
        shutil.make_archive(output[:-4], 'zip', tmp_dir)
        shutil.rmtree(tmp_dir)
    else:
        print("moving " + str(tmp_dir) + " to " + out_basepath)
        os.rename(tmp_dir, out_basepath)

Write out the database according to the GTFS format .


def _remove_I_columns(df):
    """
    Remove columns ending with I from a pandas.DataFrame

    Parameters
    ----------
    df: dataFrame

    Returns
    -------
    None
    """
    all_columns = list(filter(lambda el: el[-2:] == "_I", df.columns))
    for column in all_columns:
        del df[column]

def _remove_I_columns ( df ) : all_columns = list ( filter ( lambda el : el [ - 2 : ] == "_I" , df . columns ) ) for column in all_columns : del df [ column ]

Remove columns ending with I from a pandas . DataFrame


def _scan_footpaths_to_departure_stop(self, connection_dep_stop, connection_dep_time, arrival_time_target):
        """ A helper method for scanning the footpaths. Updates self._stop_profiles accordingly"""
        for _, neighbor, data in self._walk_network.edges_iter(nbunch=[connection_dep_stop],
                                                               data=True):
            d_walk = data['d_walk']
            neighbor_dep_time = connection_dep_time - d_walk / self._walk_speed
            pt = LabelTimeSimple(departure_time=neighbor_dep_time, arrival_time_target=arrival_time_target)
            self._stop_profiles[neighbor].update_pareto_optimal_tuples(pt)

def _scan_footpaths_to_departure_stop ( self , connection_dep_stop , connection_dep_time , arrival_time_target ) : for _ , neighbor , data in self . _walk_network . edges_iter ( nbunch = [ connection_dep_stop ] , data = True ) : d_walk = data [ 'd_walk' ] neighbor_dep_time = connection_dep_time - d_walk / self . _walk_speed pt = LabelTimeSimple ( departure_time = neighbor_dep_time , arrival_time_target = arrival_time_target ) self . _stop_profiles [ neighbor ] . update_pareto_optimal_tuples ( pt )

A helper method for scanning the footpaths . Updates self . _stop_profiles accordingly


def create_file(fname=None, fname_tmp=None, tmpdir=None,
                save_tmpfile=False, keepext=False):
    """Context manager for making files with possibility of failure.

    If you are creating a file, it is possible that the code will fail
    and leave a corrupt intermediate file.  This is especially damaging
    if this is used as automatic input to another process.  This context
    manager helps by creating a temporary filename, your code runs and
    creates that temporary file, and then if no exceptions are raised,
    the context manager will move the temporary file to the original
    filename you intended to open.

    Parameters
    ----------
    fname : str
        Target filename, this file will be created if all goes well
    fname_tmp : str
        If given, this is used as the temporary filename.
    tmpdir : str or bool
        If given, put temporary files in this directory.  If `True`,
        then find a good tmpdir that is not on local filesystem.
    save_tmpfile : bool
        If true, the temporary file is not deleteted if an exception
        is raised.
    keepext : bool, default False
            If true, have tmpfile have same extension as final file.

    Returns (as context manager value)
    ----------------------------------
     fname_tmp: str
        Temporary filename to be used.  Same as `fname_tmp`
        if given as an argument.

    Raises
    ------
    Re-raises any except occuring during the context block.
    """
    # Do nothing if requesting sqlite memory DB.
    if fname == ':memory:':
        yield fname
        return
    if fname_tmp is None:
        # no tmpfile name given - compute some basic info
        basename = os.path.basename(fname)
        root, ext = os.path.splitext(basename)
        dir_ = this_dir = os.path.dirname(fname)
        # Remove filename extension, in case this matters for
        # automatic things itself.
        if not keepext:
            root = root + ext
            ext = ''
        if tmpdir:
            # we should use a different temporary directory
            if tmpdir is True:
                # Find a directory ourself, searching some common
                # places.
                for dir__ in possible_tmpdirs:
                    if os.access(dir__, os.F_OK):
                        dir_ = dir__
                        break
        # Make the actual tmpfile, with our chosen tmpdir, directory,
        # extension.  Set it to not delete automatically, since on
        # success we will move it to elsewhere.
        tmpfile = tempfile.NamedTemporaryFile(
            prefix='tmp-' + root + '-', suffix=ext, dir=dir_, delete=False)
        fname_tmp = tmpfile.name
    try:
        yield fname_tmp
    except Exception as e:
        if save_tmpfile:
            print("Temporary file is '%s'" % fname_tmp)
        else:
            os.unlink(fname_tmp)
        raise
    # Move the file back to the original location.
    try:
        os.rename(fname_tmp, fname)
        # We have to manually set permissions.  tempfile does not use
        # umask, for obvious reasons.
        os.chmod(fname, 0o777 & ~current_umask)
    # 'Invalid cross-device link' - you can't rename files across
    # filesystems.  So, we have to fallback to moving it.  But, we
    # want to move it using tmpfiles also, so that the final file
    # appearing is atomic.  We use... tmpfiles.
    except OSError as e:
        # New temporary file in same directory
        tmpfile2 = tempfile.NamedTemporaryFile(
            prefix='tmp-' + root + '-', suffix=ext, dir=this_dir, delete=False)
        # Copy contents over
        shutil.copy(fname_tmp, tmpfile2.name)
        # Rename new tmpfile, unlink old one on other filesystem.
        os.rename(tmpfile2.name, fname)
        os.chmod(fname, 0o666 & ~current_umask)
        os.unlink(fname_tmp)

Context manager for making files with possibility of failure .


def execute(cur, *args):
    """Utility function to print sqlite queries before executing.

    Use instead of cur.execute().  First argument is cursor.

    cur.execute(stmt)
    becomes
    util.execute(cur, stmt)
    """
    stmt = args[0]
    if len(args) > 1:
        stmt = stmt.replace('%', '%%').replace('?', '%r')
        print(stmt % (args[1]))
    return cur.execute(*args)

def execute ( cur , * args ) : stmt = args [ 0 ] if len ( args ) > 1 : stmt = stmt . replace ( '%' , '%%' ) . replace ( '?' , '%r' ) print ( stmt % ( args [ 1 ] ) ) return cur . execute ( * args )

Utility function to print sqlite queries before executing .


def makedirs(path):
    """
    Create directories if they do not exist, otherwise do nothing.

    Return path for convenience
    """
    if not os.path.isdir(path):
        os.makedirs(path)
    return path

def makedirs ( path ) : if not os . path . isdir ( path ) : os . makedirs ( path ) return path

Create directories if they do not exist otherwise do nothing .


def _finalize_profiles(self):
        """
        Deal with the first walks by joining profiles to other stops within walking distance.
        """
        for stop, stop_profile in self._stop_profiles.items():
            assert (isinstance(stop_profile, NodeProfileMultiObjective))
            neighbor_label_bags = []
            walk_durations_to_neighbors = []
            departure_arrival_stop_pairs = []
            if stop_profile.get_walk_to_target_duration() != 0 and stop in self._walk_network.node:
                neighbors = networkx.all_neighbors(self._walk_network, stop)
                for neighbor in neighbors:
                    neighbor_profile = self._stop_profiles[neighbor]
                    assert (isinstance(neighbor_profile, NodeProfileMultiObjective))
                    neighbor_real_connection_labels = neighbor_profile.get_labels_for_real_connections()
                    neighbor_label_bags.append(neighbor_real_connection_labels)
                    walk_durations_to_neighbors.append(int(self._walk_network.get_edge_data(stop, neighbor)["d_walk"] /
                                                       self._walk_speed))
                    departure_arrival_stop_pairs.append((stop, neighbor))
            stop_profile.finalize(neighbor_label_bags, walk_durations_to_neighbors, departure_arrival_stop_pairs)

def _finalize_profiles ( self ) : for stop , stop_profile in self . _stop_profiles . items ( ) : assert ( isinstance ( stop_profile , NodeProfileMultiObjective ) ) neighbor_label_bags = [ ] walk_durations_to_neighbors = [ ] departure_arrival_stop_pairs = [ ] if stop_profile . get_walk_to_target_duration ( ) != 0 and stop in self . _walk_network . node : neighbors = networkx . all_neighbors ( self . _walk_network , stop ) for neighbor in neighbors : neighbor_profile = self . _stop_profiles [ neighbor ] assert ( isinstance ( neighbor_profile , NodeProfileMultiObjective ) ) neighbor_real_connection_labels = neighbor_profile . get_labels_for_real_connections ( ) neighbor_label_bags . append ( neighbor_real_connection_labels ) walk_durations_to_neighbors . append ( int ( self . _walk_network . get_edge_data ( stop , neighbor ) [ "d_walk" ] / self . _walk_speed ) ) departure_arrival_stop_pairs . append ( ( stop , neighbor ) ) stop_profile . finalize ( neighbor_label_bags , walk_durations_to_neighbors , departure_arrival_stop_pairs )

Deal with the first walks by joining profiles to other stops within walking distance .


def validate_day_start_ut(conn):
    """This validates the day_start_ut of the days table."""
    G = GTFS(conn)
    cur = conn.execute('SELECT date, day_start_ut FROM days')
    for date, day_start_ut in cur:
        #print date, day_start_ut
        assert day_start_ut == G.get_day_start_ut(date)

def
validate_day_start_ut
(
conn
)
:
G
=
GTFS
(
conn
)
cur
=
conn
.
execute
(
'SELECT date, day_start_ut FROM days'
)
for
date
,
day_start_ut
in
cur
:
#print date, day_start_ut
assert
day_start_ut
==
G
.
get_day_start_ut
(
date
)

This validates the day_start_ut of the days table .


def main_make_views(gtfs_fname):
    """Re-create all views.
    """
    print("creating views")
    conn = GTFS(fname_or_conn=gtfs_fname).conn
    for L in Loaders:
        L(None).make_views(conn)
    conn.commit()

def
main_make_views
(
gtfs_fname
)
:
print
(
"creating views"
)
conn
=
GTFS
(
fname_or_conn
=
gtfs_fname
)
.
conn
for
L
in
Loaders
:
L
(
)
.
make_views
(
conn
)
conn
.
commit
(
)


def _validate_no_null_values(self):
        """
        Loads the tables from the gtfs object and counts the number of rows that have null values in
        fields that should not be null. Stores the number of null rows in warnings_container
        """
        for table in DB_TABLE_NAMES:
            null_not_ok_warning = "Null values in must-have columns in table {table}".format(table=table)
            null_warn_warning = "Null values in good-to-have columns in table {table}".format(table=table)
            null_not_ok_fields = DB_TABLE_NAME_TO_FIELDS_WHERE_NULL_NOT_OK[table]
            null_warn_fields = DB_TABLE_NAME_TO_FIELDS_WHERE_NULL_OK_BUT_WARN[table]

            # CW, TODO: make this validation source by source
            df = self.gtfs.get_table(table)

            for warning, fields in zip([null_not_ok_warning, null_warn_warning], [null_not_ok_fields, null_warn_fields]):
                null_unwanted_df = df[fields]
                rows_having_null = null_unwanted_df.isnull().any(1)
                if sum(rows_having_null) > 0:
                    rows_having_unwanted_null = df[rows_having_null.values]
                    self.warnings_container.add_warning(warning, rows_having_unwanted_null, len(rows_having_unwanted_null))

Loads the tables from the gtfs object and counts the number of rows that have null values in fields that should not be null . Stores the number of null rows in warnings_container


def _validate_danglers(self):
        """
        Checks for rows that are not referenced in the the tables that should be linked

        stops <> stop_times using stop_I
        stop_times <> trips <> days, using trip_I
        trips <> routes, using route_I
        :return:
        """
        for query, warning in zip(DANGLER_QUERIES, DANGLER_WARNINGS):
            dangler_count = self.gtfs.execute_custom_query(query).fetchone()[0]
            if dangler_count > 0:
                if self.verbose:
                    print(str(dangler_count) + " " + warning)
                self.warnings_container.add_warning(warning, self.location, count=dangler_count)

def
_validate_danglers
(
self
)
:
for
query
,
warning
in
zip
(
DANGLER_QUERIES
,
DANGLER_WARNINGS
)
:
dangler_count
=
self
.
gtfs
.
execute_custom_query
(
query
)
.
fetchone
(
)
[
0
]
if
dangler_count
>
0
:
if
self
.
verbose
:
print
(
str
(
dangler_count
)
+
" "
+
warning
)
self
.
warnings_container
.
add_warning
(
warning
,
self
.
location
,
count
=
dangler_count
)


def print_coords(rows, prefix=''):
    """Print coordinates within a sequence.

    This is only used for debugging.  Printed in a form that can be
    pasted into Python for visualization."""
    lat = [row['lat'] for row in rows]
    lon = [row['lon'] for row in rows]
    print('COORDS'+'-' * 5)
    print("%slat, %slon = %r, %r" % (prefix, prefix, lat, lon))
    print('-'*5)

def
print_coords
(
rows
,
prefix
=
''
)
:
lat
=
[
row
[
'lat'
]
for
row
in
rows
]
lon
=
[
row
[
'lon'
]
for
row
in
rows
]
print
(
'COORDS'
+
'-'
*
5
)
print
(
"%slat, %slon = %r, %r"
%
(
prefix
,
prefix
,
lat
,
lon
)
)
print
(
'-'
*
5
)


def find_segments(stops, shape):
    """Find corresponding shape points for a list of stops and create shape break points.

    Parameters
    ----------
    stops: stop-sequence (list)
        List of stop points
    shape: list of shape points
        shape-sequence of shape points

    Returns
    -------
    break_points: list[int]
        stops[i] corresponds to shape[break_points[i]].  This list can
        be used to partition the shape points into segments between
        one stop and the next.
    badness: float
        Lower indicates better fit to the shape.  This is the sum of
        distances (in meters) between every each stop and its closest
        shape point.  This is not needed in normal use, but in the
        cases where you must determine the best-fitting shape for a
        stop-sequence, use this.
    """
    if not shape:
        return [], 0
    break_points = []
    last_i = 0
    cumul_d = 0
    badness = 0
    d_last_stop = float('inf')
    lstlat, lstlon = None, None
    break_shape_points = []
    for stop in stops:
        stlat, stlon = stop['lat'], stop['lon']
        best_d = float('inf')
        # print stop
        if badness > 500 and badness > 30 * len(break_points):
            return [], badness
        for i in range(last_i, len(shape)):
            d = wgs84_distance(stlat, stlon, shape[i]['lat'], shape[i]['lon'])
            if lstlat:
                d_last_stop = wgs84_distance(lstlat, lstlon, shape[i]['lat'], shape[i]['lon'])
            # If we are getting closer to next stop, record this as
            # the best stop so far.continue
            if d < best_d:
                best_d = d
                best_i = i
                # print best_d, i, last_i, len(shape)
                cumul_d += d
            # We have to be very careful about our stop condition.
            # This is trial and error, basically.
            if (d_last_stop < d) or (d > 500) or (i < best_i + 100):
                    continue
            # We have decided our best stop, stop looking and continue
            # the outer loop.
            else:
                badness += best_d
                break_points.append(best_i)
                last_i = best_i
                lstlat, lstlon = stlat, stlon
                break_shape_points.append(shape[best_i])
                break
        else:
            # Executed if we did *not* break the inner loop
            badness += best_d
            break_points.append(best_i)
            last_i = best_i
            lstlat, lstlon = stlat, stlon
            break_shape_points.append(shape[best_i])
            pass
    # print "Badness:", badness
    # print_coords(stops, 'stop')
    # print_coords(shape, 'shape')
    # print_coords(break_shape_points, 'break')
    return break_points, badness

Find corresponding shape points for a list of stops and create shape break points .


def return_segments(shape, break_points):
    """Break a shape into segments between stops using break_points.

    This function can use the `break_points` outputs from
    `find_segments`, and cuts the shape-sequence into pieces
    corresponding to each stop.
    """
    # print 'xxx'
    # print stops
    # print shape
    # print break_points
    # assert len(stops) == len(break_points)
    segs = []
    bp = 0 # not used
    bp2 = 0
    for i in range(len(break_points)-1):
        bp = break_points[i] if break_points[i] is not None else bp2
        bp2 = break_points[i+1] if break_points[i+1] is not None else bp
        segs.append(shape[bp:bp2+1])
    segs.append([])
    return segs

Break a shape into segments between stops using break_points .


def get_trip_points(cur, route_id, offset=0, tripid_glob=''):
    """Get all scheduled stops on a particular route_id.

    Given a route_id, return the trip-stop-list with
    latitude/longitudes.  This is a bit more tricky than it seems,
    because we have to go from table route->trips->stop_times.  This
    functions finds an arbitrary trip (in trip table) with this route ID
    and, and then returns all stop points for that trip.

    Parameters
    ----------
    cur : sqlite3.Cursor
        cursor to sqlite3 DB containing GTFS
    route_id : string or any
        route_id to get stop points of
    offset : int
        LIMIT offset if you don't want the first trip returned.
    tripid_glob : string
        If given, allows you to limit tripids which can be selected.
        Mainly useful in debugging.

    Returns
    -------
    stop-list
        List of stops in stop-seq format.
    """
    extra_where = ''
    if tripid_glob:
        extra_where = "AND trip_id GLOB '%s'" % tripid_glob
    cur.execute('SELECT seq, lat, lon '
                'FROM (select trip_I from route '
                '      LEFT JOIN trips USING (route_I) '
                '      WHERE route_id=? %s limit 1 offset ? ) '
                'JOIN stop_times USING (trip_I) '
                'LEFT JOIN stop USING (stop_id) '
                'ORDER BY seq' % extra_where, (route_id, offset))
    stop_points = [dict(seq=row[0], lat=row[1], lon=row[2]) for row in cur]
    return stop_points

'      LEFT JOIN trips USING (route_I) '

Get all scheduled stops on a particular route_id .


def interpolate_shape_times(shape_distances, shape_breaks, stop_times):
    """
    Interpolate passage times for shape points.

    Parameters
    ----------
    shape_distances: list
        list of cumulative distances along the shape
    shape_breaks: list
        list of shape_breaks
    stop_times: list
        list of stop_times

    Returns
    -------
    shape_times: list of ints (seconds) / numpy array
        interpolated shape passage times

    The values of stop times before the first shape-break are given the first
    stopping time, and the any shape points after the last break point are
    given the value of the last shape point.
    """
    shape_times = np.zeros(len(shape_distances))
    shape_times[:shape_breaks[0]] = stop_times[0]
    for i in range(len(shape_breaks)-1):
        cur_break = shape_breaks[i]
        cur_time = stop_times[i]
        next_break = shape_breaks[i+1]
        next_time = stop_times[i+1]
        if cur_break == next_break:
            shape_times[cur_break] = stop_times[i]
        else:
            cur_distances = shape_distances[cur_break:next_break+1]
            norm_distances = ((np.array(cur_distances)-float(cur_distances[0])) /
                              float(cur_distances[-1] - cur_distances[0]))
            times = (1.-norm_distances)*cur_time+norm_distances*next_time
            shape_times[cur_break:next_break] = times[:-1]
    # deal final ones separately:
    shape_times[shape_breaks[-1]:] = stop_times[-1]
    return list(shape_times)

def
interpolate_shape_times
(
shape_distances
,
shape_breaks
,
stop_times
)
:
shape_times
=
np
.
zeros
(
len
(
shape_distances
)
)
shape_times
[
:
shape_breaks
[
0
]
]
=
stop_times
[
0
]
for
i
in
range
(
len
(
shape_breaks
)
1
)
:
cur_break
=
shape_breaks
[
i
]
cur_time
=
stop_times
[
i
]
next_break
=
shape_breaks
[
i
+
1
]
next_time
=
stop_times
[
i
+
1
]
if
cur_break
==
next_break
:
shape_times
[
cur_break
]
=
stop_times
[
i
]
else
:
cur_distances
=
shape_distances
[
cur_break
:
next_break
+
1
]
norm_distances
=
(
(
np
.
array
(
cur_distances
)
float
(
cur_distances
[
0
]
)
)
/
float
(
cur_distances
[
1
]
cur_distances
[
0
]
)
)
times
=
(
1.
norm_distances
)
*
cur_time
+
norm_distances
*
next_time
shape_times
[
cur_break
:
next_break
]
=
times
[
:
1
]
# deal final ones separately:
shape_times
[
shape_breaks
[
1
]
:
]
=
stop_times
[
1
]
return
list
(
shape_times
)

Interpolate passage times for shape points .


def evaluate_earliest_arrival_time_at_target(self, dep_time, transfer_margin):
        """
        Get the earliest arrival time at the target, given a departure time.

        Parameters
        ----------
        dep_time : float, int
            time in unix seconds
        transfer_margin: float, int
            transfer margin in seconds

        Returns
        -------
        arrival_time : float
            Arrival time in the given time unit (seconds after unix epoch).
        """
        minimum = dep_time + self._walk_to_target_duration
        dep_time_plus_transfer_margin = dep_time + transfer_margin
        for label in self._labels:
            if label.departure_time >= dep_time_plus_transfer_margin and label.arrival_time_target < minimum:
                minimum = label.arrival_time_target
        return float(minimum)

Get the earliest arrival time at the target given a departure time .


def _run(self):
        """
        Run the actual simulation.
        """
        if self._has_run:
            raise RuntimeError("This spreader instance has already been run: "
                               "create a new Spreader object for a new run.")
        i = 1
        while self.event_heap.size() > 0 and len(self._uninfected_stops) > 0:
            event = self.event_heap.pop_next_event()
            this_stop = self._stop_I_to_spreading_stop[event.from_stop_I]

            if event.arr_time_ut > self.start_time_ut + self.max_duration_ut:
                break

            if this_stop.can_infect(event):

                target_stop = self._stop_I_to_spreading_stop[event.to_stop_I]
                already_visited = target_stop.has_been_visited()
                target_stop.visit(event)

                if not already_visited:
                    self._uninfected_stops.remove(event.to_stop_I)
                    print(i, self.event_heap.size())
                    transfer_distances = self.gtfs.get_straight_line_transfer_distances(event.to_stop_I)
                    self.event_heap.add_walk_events_to_heap(transfer_distances, event, self.start_time_ut,
                                                            self.walk_speed, self._uninfected_stops,
                                                            self.max_duration_ut)
                    i += 1
        self._has_run = True


def add_walk_distances_to_db_python(gtfs, osm_path, cutoff_distance_m=1000):
    """
    Computes the walk paths between stops, and updates these to the gtfs database.

    Parameters
    ----------
    gtfs: gtfspy.GTFS or str
        A GTFS object or a string representation.
    osm_path: str
        path to the OpenStreetMap file
    cutoff_distance_m: number
        maximum allowed distance in meters

    Returns
    -------
    None

    See Also
    --------
    gtfspy.calc_transfers
    compute_walk_paths_java
    """
    if isinstance(gtfs, str):
        gtfs = GTFS(gtfs)
    assert (isinstance(gtfs, GTFS))
    print("Reading in walk network")
    walk_network = create_walk_network_from_osm(osm_path)
    print("Matching stops to the OSM network")
    stop_I_to_nearest_osm_node, stop_I_to_nearest_osm_node_distance = match_stops_to_nodes(gtfs, walk_network)

    transfers = gtfs.get_straight_line_transfer_distances()

    from_I_to_to_stop_Is = {stop_I: set() for stop_I in stop_I_to_nearest_osm_node}
    for transfer_tuple in transfers.itertuples():
        from_I = transfer_tuple.from_stop_I
        to_I = transfer_tuple.to_stop_I
        from_I_to_to_stop_Is[from_I].add(to_I)

    print("Computing walking distances")
    for from_I, to_stop_Is in from_I_to_to_stop_Is.items():
        from_node = stop_I_to_nearest_osm_node[from_I]
        from_dist = stop_I_to_nearest_osm_node_distance[from_I]
        shortest_paths = networkx.single_source_dijkstra_path_length(walk_network,
                                                                     from_node,
                                                                     cutoff=cutoff_distance_m - from_dist,
                                                                     weight="distance")
        for to_I in to_stop_Is:
            to_distance = stop_I_to_nearest_osm_node_distance[to_I]
            to_node = stop_I_to_nearest_osm_node[to_I]
            osm_distance = shortest_paths.get(to_node, float('inf'))
            total_distance = from_dist + osm_distance + to_distance
            from_stop_I_transfers = transfers[transfers['from_stop_I'] == from_I]
            straigth_distance = from_stop_I_transfers[from_stop_I_transfers["to_stop_I"] == to_I]["d"].values[0]
            assert (straigth_distance < total_distance + 2)  # allow for a maximum  of 2 meters in calculations
            if total_distance <= cutoff_distance_m:
                gtfs.conn.execute("UPDATE stop_distances "
                                  "SET d_walk = " + str(int(total_distance)) +
                                  " WHERE from_stop_I=" + str(from_I) + " AND to_stop_I=" + str(to_I))

    gtfs.conn.commit()

Computes the walk paths between stops and updates these to the gtfs database .


def stop_to_stop_network_for_route_type(gtfs,
                                        route_type,
                                        link_attributes=None,
                                        start_time_ut=None,
                                        end_time_ut=None):
    """
    Get a stop-to-stop network describing a single mode of travel.

    Parameters
    ----------
    gtfs : gtfspy.GTFS
    route_type : int
        See gtfspy.route_types.TRANSIT_ROUTE_TYPES for the list of possible types.
    link_attributes: list[str], optional
        defaulting to use the following link attributes:
            "n_vehicles" : Number of vehicles passed
            "duration_min" : minimum travel time between stops
            "duration_max" : maximum travel time between stops
            "duration_median" : median travel time between stops
            "duration_avg" : average travel time between stops
            "d" : distance along straight line (wgs84_distance)
            "distance_shape" : minimum distance along shape
            "capacity_estimate" : approximate capacity passed through the stop
            "route_I_counts" : dict from route_I to counts
    start_time_ut: int
        start time of the time span (in unix time)
    end_time_ut: int
        end time of the time span (in unix time)

    Returns
    -------
    net: networkx.DiGraph
        A directed graph Directed graph
    """
    if link_attributes is None:
        link_attributes = DEFAULT_STOP_TO_STOP_LINK_ATTRIBUTES
    assert(route_type in route_types.TRANSIT_ROUTE_TYPES)

    stops_dataframe = gtfs.get_stops_for_route_type(route_type)
    net = networkx.DiGraph()
    _add_stops_to_net(net, stops_dataframe)

    events_df = gtfs.get_transit_events(start_time_ut=start_time_ut,
                                        end_time_ut=end_time_ut,
                                        route_type=route_type)
    if len(net.nodes()) < 2:
        assert events_df.shape[0] == 0

    # group events by links, and loop over them (i.e. each link):
    link_event_groups = events_df.groupby(['from_stop_I', 'to_stop_I'], sort=False)
    for key, link_events in link_event_groups:
        from_stop_I, to_stop_I = key
        assert isinstance(link_events, pd.DataFrame)
        # 'dep_time_ut' 'arr_time_ut' 'shape_id' 'route_type' 'trip_I' 'duration' 'from_seq' 'to_seq'
        if link_attributes is None:
            net.add_edge(from_stop_I, to_stop_I)
        else:
            link_data = {}
            if "duration_min" in link_attributes:
                link_data['duration_min'] = float(link_events['duration'].min())
            if "duration_max" in link_attributes:
                link_data['duration_max'] = float(link_events['duration'].max())
            if "duration_median" in link_attributes:
                link_data['duration_median'] = float(link_events['duration'].median())
            if "duration_avg" in link_attributes:
                link_data['duration_avg'] = float(link_events['duration'].mean())
            # statistics on numbers of vehicles:
            if "n_vehicles" in link_attributes:
                link_data['n_vehicles'] = int(link_events.shape[0])
            if "capacity_estimate" in link_attributes:
                link_data['capacity_estimate'] = route_types.ROUTE_TYPE_TO_APPROXIMATE_CAPACITY[route_type] \
                                                 * int(link_events.shape[0])
            if "d" in link_attributes:
                from_lat = net.node[from_stop_I]['lat']
                from_lon = net.node[from_stop_I]['lon']
                to_lat = net.node[to_stop_I]['lat']
                to_lon = net.node[to_stop_I]['lon']
                distance = wgs84_distance(from_lat, from_lon, to_lat, to_lon)
                link_data['d'] = int(distance)
            if "distance_shape" in link_attributes:
                assert "shape_id" in link_events.columns.values
                found = None
                for i, shape_id in enumerate(link_events["shape_id"].values):
                    if shape_id is not None:
                        found = i
                        break
                if found is None:
                    link_data["distance_shape"] = None
                else:
                    link_event = link_events.iloc[found]
                    distance = gtfs.get_shape_distance_between_stops(
                        link_event["trip_I"],
                        int(link_event["from_seq"]),
                        int(link_event["to_seq"])
                    )
                    link_data['distance_shape'] = distance
            if "route_I_counts" in link_attributes:
                link_data["route_I_counts"] = link_events.groupby("route_I").size().to_dict()
            net.add_edge(from_stop_I, to_stop_I, attr_dict=link_data)
    return net

Get a stop - to - stop network describing a single mode of travel .


def combined_stop_to_stop_transit_network(gtfs, start_time_ut=None, end_time_ut=None):
    """
    Compute stop-to-stop networks for all travel modes and combine them into a single network.
    The modes of transport are encoded to a single network.
    The network consists of multiple links corresponding to each travel mode.
    Walk mode is not included.

    Parameters
    ----------
    gtfs: gtfspy.GTFS

    Returns
    -------
    net: networkx.MultiDiGraph
        keys should be one of route_types.TRANSIT_ROUTE_TYPES (i.e. GTFS route_types)
    """
    multi_di_graph = networkx.MultiDiGraph()
    for route_type in route_types.TRANSIT_ROUTE_TYPES:
        graph = stop_to_stop_network_for_route_type(gtfs, route_type,
                                                    start_time_ut=start_time_ut, end_time_ut=end_time_ut)
        for from_node, to_node, data in graph.edges(data=True):
            data['route_type'] = route_type
        multi_di_graph.add_edges_from(graph.edges(data=True))
        multi_di_graph.add_nodes_from(graph.nodes(data=True))
    return multi_di_graph

def combined_stop_to_stop_transit_network ( gtfs , start_time_ut = None , end_time_ut = None ) : multi_di_graph = networkx . MultiDiGraph ( ) for route_type in route_types . TRANSIT_ROUTE_TYPES : graph = stop_to_stop_network_for_route_type ( gtfs , route_type , start_time_ut = start_time_ut , end_time_ut = end_time_ut ) for from_node , to_node , data in graph . edges ( data = True ) : data [ 'route_type' ] = route_type multi_di_graph . add_edges_from ( graph . edges ( data = True ) ) multi_di_graph . add_nodes_from ( graph . nodes ( data = True ) ) return multi_di_graph

Compute stop - to - stop networks for all travel modes and combine them into a single network . The modes of transport are encoded to a single network . The network consists of multiple links corresponding to each travel mode . Walk mode is not included .


def temporal_network(gtfs,
                     start_time_ut=None,
                     end_time_ut=None,
                     route_type=None):
    """
    Compute the temporal network of the data, and return it as a pandas.DataFrame

    Parameters
    ----------
    gtfs : gtfspy.GTFS
    start_time_ut: int | None
        start time of the time span (in unix time)
    end_time_ut: int | None
        end time of the time span (in unix time)
    route_type: int | None
        Specifies which mode of public transport are included, or whether all modes should be included.
        The int should be one of the standard GTFS route_types:
        (see also gtfspy.route_types.TRANSIT_ROUTE_TYPES )
        If route_type is not specified, all modes are included.

    Returns
    -------
    events_df: pandas.DataFrame
        Columns: departure_stop, arrival_stop, departure_time_ut, arrival_time_ut, route_type, route_I, trip_I
    """
    events_df = gtfs.get_transit_events(start_time_ut=start_time_ut,
                                        end_time_ut=end_time_ut,
                                        route_type=route_type)
    events_df.drop('to_seq', 1, inplace=True)
    events_df.drop('shape_id', 1, inplace=True)
    events_df.drop('duration', 1, inplace=True)
    events_df.drop('route_id', 1, inplace=True)
    events_df.rename(
        columns={
            'from_seq': "seq"
        },
        inplace=True
    )
    return events_df

def temporal_network ( gtfs , start_time_ut = None , end_time_ut = None , route_type = None ) : events_df = gtfs . get_transit_events ( start_time_ut = start_time_ut , end_time_ut = end_time_ut , route_type = route_type ) events_df . drop ( 'to_seq' , 1 , inplace = True ) events_df . drop ( 'shape_id' , 1 , inplace = True ) events_df . drop ( 'duration' , 1 , inplace = True ) events_df . drop ( 'route_id' , 1 , inplace = True ) events_df . rename ( columns = { 'from_seq' : "seq" } , inplace = True ) return events_df

Compute the temporal network of the data and return it as a pandas . DataFrame


def plot_temporal_distance_cdf(self):
        """
        Plot the temporal distance cumulative density function.

        Returns
        -------
        fig: matplotlib.Figure
        """
        xvalues, cdf = self.profile_block_analyzer._temporal_distance_cdf()
        fig = plt.figure()
        ax = fig.add_subplot(111)
        xvalues = numpy.array(xvalues) / 60.0
        ax.plot(xvalues, cdf, "-k")
        ax.fill_between(xvalues, cdf, color="red", alpha=0.2)
        ax.set_ylabel("CDF(t)")
        ax.set_xlabel("Temporal distance t (min)")
        return fig

def
plot_temporal_distance_cdf
(
self
)
:
xvalues
,
cdf
=
self
.
profile_block_analyzer
.
_temporal_distance_cdf
(
)
fig
=
plt
.
figure
(
)
ax
=
fig
.
add_subplot
(
111
)
xvalues
=
numpy
.
array
(
xvalues
)
/
60.0
ax
.
plot
(
xvalues
,
cdf
,
"-k"
)
ax
.
fill_between
(
xvalues
,
cdf
,
color
=
"red"
,
alpha
=
0.2
)
ax
.
set_ylabel
(
"CDF(t)"
)
ax
.
set_xlabel
(
"Temporal distance t (min)"
)
return
fig

Plot the temporal distance cumulative density function .


def get_transfer_stop_pairs(self):
        """
        Get stop pairs through which transfers take place

        Returns
        -------
        transfer_stop_pairs: list
        """
        transfer_stop_pairs = []
        previous_arrival_stop = None
        current_trip_id = None
        for leg in self.legs:
            if leg.trip_id is not None and leg.trip_id != current_trip_id and previous_arrival_stop is not None:
                transfer_stop_pair = (previous_arrival_stop, leg.departure_stop)
                transfer_stop_pairs.append(transfer_stop_pair)
            previous_arrival_stop = leg.arrival_stop
            current_trip_id = leg.trip_id
        return transfer_stop_pairs

def get_transfer_stop_pairs ( self ) : transfer_stop_pairs = [ ] previous_arrival_stop = None current_trip_id = None for leg in self . legs : if leg . trip_id is not None and leg . trip_id != current_trip_id and previous_arrival_stop is not None : transfer_stop_pair = ( previous_arrival_stop , leg . departure_stop ) transfer_stop_pairs . append ( transfer_stop_pair ) previous_arrival_stop = leg . arrival_stop current_trip_id = leg . trip_id return transfer_stop_pairs


def from_directory_as_inmemory_db(cls, gtfs_directory):
        """
        Instantiate a GTFS object by computing

        Parameters
        ----------
        gtfs_directory: str
            path to the directory for importing the database
        """
        # this import is here to avoid circular imports (which turned out to be a problem)
        from gtfspy.import_gtfs import import_gtfs
        conn = sqlite3.connect(":memory:")
        import_gtfs(gtfs_directory,
                    conn,
                    preserve_connection=True,
                    print_progress=False)
        return cls(conn)


def get_main_database_path(self):
        """
        Should return the path to the database

        Returns
        -------
        path : unicode
            path to the database, empty string for in-memory databases
        """
        cur = self.conn.cursor()
        cur.execute("PRAGMA database_list")
        rows = cur.fetchall()
        for row in rows:
            if row[1] == str("main"):
                return row[2]

def
get_main_database_path
(
self
)
:
cur
=
self
.
conn
.
cursor
(
)
cur
.
execute
(
"PRAGMA database_list"
)
rows
=
cur
.
fetchall
(
)
for
row
in
rows
:
if
row
[
1
]
==
str
(
"main"
)
:
return
row
[
2
]


def get_shape_distance_between_stops(self, trip_I, from_stop_seq, to_stop_seq):
        """
        Get the distance along a shape between stops

        Parameters
        ----------
        trip_I : int
            trip_ID along which we travel
        from_stop_seq : int
            the sequence number of the 'origin' stop
        to_stop_seq : int
            the sequence number of the 'destination' stop

        Returns
        -------
        distance : float, None
            If the shape calculation succeeded, return a float, otherwise return None
            (i.e. in the case where the shapes table is empty)
        """

        query_template = "SELECT shape_break FROM stop_times WHERE trip_I={trip_I} AND seq={seq} "
        stop_seqs = [from_stop_seq, to_stop_seq]
        shape_breaks = []
        for seq in stop_seqs:
            q = query_template.format(seq=seq, trip_I=trip_I)
            shape_breaks.append(self.conn.execute(q).fetchone())
        query_template = "SELECT max(d) - min(d) " \
                         "FROM shapes JOIN trips ON(trips.shape_id=shapes.shape_id) " \
                         "WHERE trip_I={trip_I} AND shapes.seq>={from_stop_seq} AND shapes.seq<={to_stop_seq};"
        distance_query = query_template.format(trip_I=trip_I, from_stop_seq=from_stop_seq, to_stop_seq=to_stop_seq)
        return self.conn.execute(distance_query).fetchone()[0]


def get_timezone_name(self):
        """
        Get name of the GTFS timezone

        Returns
        -------
        timezone_name : str
            name of the time zone, e.g. "Europe/Helsinki"
        """
        tz_name = self.conn.execute('SELECT timezone FROM agencies LIMIT 1').fetchone()
        if tz_name is None:
            raise ValueError("This database does not have a timezone defined.")
        return tz_name[0]


def get_trip_trajectories_within_timespan(self, start, end, use_shapes=True, filter_name=None):
        """
        Get complete trip data for visualizing public transport operation based on gtfs.

        Parameters
        ----------
        start: number
            Earliest position data to return (in unix time)
        end: number
            Latest position data to return (in unix time)
        use_shapes: bool, optional
            Whether or not shapes should be included
        filter_name: str
            Pick only routes having this name.

        Returns
        -------
        trips: dict
            trips['trips'] is a list whose each element (e.g. el = trips['trips'][0])
            is a dict with the following properties:
                el['lats'] -- list of latitudes
                el['lons'] -- list of longitudes
                el['times'] -- list of passage_times
                el['route_type'] -- type of vehicle as specified by GTFS
                el['name'] -- name of the route
        """
        trips = []
        trip_df = self.get_tripIs_active_in_range(start, end)
        print("gtfs_viz.py: fetched " + str(len(trip_df)) + " trip ids")
        shape_cache = {}

        # loop over all trips:
        for row in trip_df.itertuples():
            trip_I = row.trip_I
            day_start_ut = row.day_start_ut
            shape_id = row.shape_id

            trip = {}

            name, route_type = self.get_route_name_and_type_of_tripI(trip_I)
            trip['route_type'] = int(route_type)
            trip['name'] = str(name)

            if filter_name and (name != filter_name):
                continue

            stop_lats = []
            stop_lons = []
            stop_dep_times = []
            shape_breaks = []
            stop_seqs = []

            # get stop_data and store it:
            stop_time_df = self.get_trip_stop_time_data(trip_I, day_start_ut)
            for stop_row in stop_time_df.itertuples():
                stop_lats.append(float(stop_row.lat))
                stop_lons.append(float(stop_row.lon))
                stop_dep_times.append(float(stop_row.dep_time_ut))
                try:
                    stop_seqs.append(int(stop_row.seq))
                except TypeError:
                    stop_seqs.append(None)
                if use_shapes:
                    try:
                        shape_breaks.append(int(stop_row.shape_break))
                    except (TypeError, ValueError):
                        shape_breaks.append(None)

            if use_shapes:
                # get shape data (from cache, if possible)
                if shape_id not in shape_cache:
                    shape_cache[shape_id] = shapes.get_shape_points2(self.conn.cursor(), shape_id)
                shape_data = shape_cache[shape_id]
                # noinspection PyBroadException
                try:
                    trip['times'] = shapes.interpolate_shape_times(shape_data['d'], shape_breaks, stop_dep_times)
                    trip['lats'] = shape_data['lats']
                    trip['lons'] = shape_data['lons']
                    start_break = shape_breaks[0]
                    end_break = shape_breaks[-1]
                    trip['times'] = trip['times'][start_break:end_break + 1]
                    trip['lats'] = trip['lats'][start_break:end_break + 1]
                    trip['lons'] = trip['lons'][start_break:end_break + 1]
                except:
                    # In case interpolation fails:
                    trip['times'] = stop_dep_times
                    trip['lats'] = stop_lats
                    trip['lons'] = stop_lons
            else:
                trip['times'] = stop_dep_times
                trip['lats'] = stop_lats
                trip['lons'] = stop_lons
            trips.append(trip)
        return {"trips": trips}

Get complete trip data for visualizing public transport operation based on gtfs .


def get_stop_count_data(self, start_ut, end_ut):
        """
        Get stop count data.

        Parameters
        ----------
        start_ut : int
            start time in unixtime
        end_ut : int
            end time in unixtime

        Returns
        -------
        stopData : pandas.DataFrame
            each row in the stopData dataFrame is a dictionary with the following elements
                stop_I, count, lat, lon, name
            with data types
                (int, int, float, float, str)
        """
        # TODO! this function could perhaps be made a single sql query now with the new tables?
        trips_df = self.get_tripIs_active_in_range(start_ut, end_ut)
        # stop_I -> count, lat, lon, name
        stop_counts = Counter()

        # loop over all trips:
        for row in trips_df.itertuples():
            # get stop_data and store it:
            stops_seq = self.get_trip_stop_time_data(row.trip_I, row.day_start_ut)
            for stop_time_row in stops_seq.itertuples(index=False):
                if (stop_time_row.dep_time_ut >= start_ut) and (stop_time_row.dep_time_ut <= end_ut):
                    stop_counts[stop_time_row.stop_I] += 1

        all_stop_data = self.stops()
        counts = [stop_counts[stop_I] for stop_I in all_stop_data["stop_I"].values]

        all_stop_data.loc[:, "count"] = pd.Series(counts, index=all_stop_data.index)
        return all_stop_data