export BrickStepperSpeedRamping
struct BrickStepperSpeedRamping
    acceleration::Integer
    deacceleration::Integer
end

export BrickStepperAllData
struct BrickStepperAllData
    current_velocity::Integer
    current_position::Integer
    remaining_steps::Integer
    stack_voltage::Integer
    external_voltage::Integer
    current_consumption::Integer
end

export BrickStepperSPITFPBaudrateConfig
struct BrickStepperSPITFPBaudrateConfig
    enable_dynamic_baudrate::Bool
    minimum_dynamic_baudrate::Integer
end

export BrickStepperSPITFPErrorCount
struct BrickStepperSPITFPErrorCount
    error_count_ack_checksum::Integer
    error_count_message_checksum::Integer
    error_count_frame::Integer
    error_count_overflow::Integer
end

export BrickStepperProtocol1BrickletName
struct BrickStepperProtocol1BrickletName
    protocol_version::Integer
    firmware_version::Vector{Integer}
    name::String
end

export BrickStepperIdentity
struct BrickStepperIdentity
    uid::String
    connected_uid::String
    position::Char
    hardware_version::Vector{Integer}
    firmware_version::Vector{Integer}
    device_identifier::Integer
end

export BrickStepper
"""
Drives one bipolar stepper motor with up to 38V and 2.5A per phase
"""
mutable struct BrickStepper <: TinkerforgeDevice
    ipcon::IPConnection
    deviceInternal::PyObject

    """
    Creates an object with the unique device ID *uid* and adds it to
    the IP Connection *ipcon*.
    """
    function BrickStepper(uid::String, ipcon::IPConnection)
        package = pyimport("tinkerforge.brick_stepper")
        deviceInternal = package.BrickStepper(uid, ipcon.ipconInternal)

        return new(ipcon, deviceInternal)
    end
end

export set_max_velocity
"""
    $(SIGNATURES)

Sets the maximum velocity of the stepper motor.
This function does *not* start the motor, it merely sets the maximum
velocity the stepper motor is accelerated to. To get the motor running use
either :func:`Set Target Position`, :func:`Set Steps`, :func:`Drive Forward` or
:func:`Drive Backward`.
"""
function set_max_velocity(device::BrickStepper, velocity)
    device.deviceInternal.set_max_velocity(velocity)
end

export get_max_velocity
"""
    $(SIGNATURES)

Returns the velocity as set by :func:`Set Max Velocity`.
"""
function get_max_velocity(device::BrickStepper)
    return device.deviceInternal.get_max_velocity()
end

export get_current_velocity
"""
    $(SIGNATURES)

Returns the *current* velocity of the stepper motor.
"""
function get_current_velocity(device::BrickStepper)
    return device.deviceInternal.get_current_velocity()
end

export set_speed_ramping
"""
    $(SIGNATURES)

Sets the acceleration and deacceleration of the stepper motor.
An acceleration of 1000 means, that
every second the velocity is increased by 1000 *steps/s*.

For example: If the current velocity is 0 and you want to accelerate to a
velocity of 8000 *steps/s* in 10 seconds, you should set an acceleration
of 800 *steps/s²*.

An acceleration/deacceleration of 0 means instantaneous
acceleration/deacceleration (not recommended)
"""
function set_speed_ramping(device::BrickStepper, acceleration, deacceleration)
    device.deviceInternal.set_speed_ramping(acceleration, deacceleration)
end

export get_speed_ramping
"""
    $(SIGNATURES)

Returns the acceleration and deacceleration as set by
:func:`Set Speed Ramping`.
"""
function get_speed_ramping(device::BrickStepper)
    return device.deviceInternal.get_speed_ramping()
end

export full_brake
"""
    $(SIGNATURES)

Executes an active full brake.

.. warning::
 This function is for emergency purposes,
 where an immediate brake is necessary. Depending on the current velocity and
 the strength of the motor, a full brake can be quite violent.

Call :func:`Stop` if you just want to stop the motor.
"""
function full_brake(device::BrickStepper)
    device.deviceInternal.full_brake()
end

export set_current_position
"""
    $(SIGNATURES)

Sets the current steps of the internal step counter. This can be used to
set the current position to 0 when some kind of starting position
is reached (e.g. when a CNC machine reaches a corner).
"""
function set_current_position(device::BrickStepper, position)
    device.deviceInternal.set_current_position(position)
end

export get_current_position
"""
    $(SIGNATURES)

Returns the current position of the stepper motor in steps. On startup
the position is 0. The steps are counted with all possible driving
functions (:func:`Set Target Position`, :func:`Set Steps`, :func:`Drive Forward` or
:func:`Drive Backward`). It also is possible to reset the steps to 0 or
set them to any other desired value with :func:`Set Current Position`.
"""
function get_current_position(device::BrickStepper)
    return device.deviceInternal.get_current_position()
end

export set_target_position
"""
    $(SIGNATURES)

Sets the target position of the stepper motor in steps. For example,
if the current position of the motor is 500 and :func:`Set Target Position` is
called with 1000, the stepper motor will drive 500 steps forward. It will
use the velocity, acceleration and deacceleration as set by
:func:`Set Max Velocity` and :func:`Set Speed Ramping`.

A call of :func:`Set Target Position` with the parameter *x* is equivalent to
a call of :func:`Set Steps` with the parameter
(*x* - :func:`Get Current Position`).
"""
function set_target_position(device::BrickStepper, position)
    device.deviceInternal.set_target_position(position)
end

export get_target_position
"""
    $(SIGNATURES)

Returns the last target position as set by :func:`Set Target Position`.
"""
function get_target_position(device::BrickStepper)
    return device.deviceInternal.get_target_position()
end

export set_steps
"""
    $(SIGNATURES)

Sets the number of steps the stepper motor should run. Positive values
will drive the motor forward and negative values backward.
The velocity, acceleration and deacceleration as set by
:func:`Set Max Velocity` and :func:`Set Speed Ramping` will be used.
"""
function set_steps(device::BrickStepper, steps)
    device.deviceInternal.set_steps(steps)
end

export get_steps
"""
    $(SIGNATURES)

Returns the last steps as set by :func:`Set Steps`.
"""
function get_steps(device::BrickStepper)
    return device.deviceInternal.get_steps()
end

export get_remaining_steps
"""
    $(SIGNATURES)

Returns the remaining steps of the last call of :func:`Set Steps`.
For example, if :func:`Set Steps` is called with 2000 and
:func:`Get Remaining Steps` is called after the motor has run for 500 steps,
it will return 1500.
"""
function get_remaining_steps(device::BrickStepper)
    return device.deviceInternal.get_remaining_steps()
end

export set_step_mode
"""
    $(SIGNATURES)

Sets the step mode of the stepper motor. Possible values are:

* Full Step = 1
* Half Step = 2
* Quarter Step = 4
* Eighth Step = 8

A higher value will increase the resolution and
decrease the torque of the stepper motor.
"""
function set_step_mode(device::BrickStepper, mode)
    device.deviceInternal.set_step_mode(mode)
end

export get_step_mode
"""
    $(SIGNATURES)

Returns the step mode as set by :func:`Set Step Mode`.
"""
function get_step_mode(device::BrickStepper)
    return device.deviceInternal.get_step_mode()
end

export drive_forward
"""
    $(SIGNATURES)

Drives the stepper motor forward until :func:`Drive Backward` or
:func:`Stop` is called. The velocity, acceleration and deacceleration as
set by :func:`Set Max Velocity` and :func:`Set Speed Ramping` will be used.
"""
function drive_forward(device::BrickStepper)
    device.deviceInternal.drive_forward()
end

export drive_backward
"""
    $(SIGNATURES)

Drives the stepper motor backward until :func:`Drive Forward` or
:func:`Stop` is triggered. The velocity, acceleration and deacceleration as
set by :func:`Set Max Velocity` and :func:`Set Speed Ramping` will be used.
"""
function drive_backward(device::BrickStepper)
    device.deviceInternal.drive_backward()
end

export stop
"""
    $(SIGNATURES)

Stops the stepper motor with the deacceleration as set by
:func:`Set Speed Ramping`.
"""
function stop(device::BrickStepper)
    device.deviceInternal.stop()
end

export get_stack_input_voltage
"""
    $(SIGNATURES)

Returns the stack input voltage. The stack input voltage is the
voltage that is supplied via the stack, i.e. it is given by a
Step-Down or Step-Up Power Supply.
"""
function get_stack_input_voltage(device::BrickStepper)
    return device.deviceInternal.get_stack_input_voltage()
end

export get_external_input_voltage
"""
    $(SIGNATURES)

Returns the external input voltage. The external input voltage is
given via the black power input connector on the Stepper Brick.

If there is an external input voltage and a stack input voltage, the motor
will be driven by the external input voltage. If there is only a stack
voltage present, the motor will be driven by this voltage.

.. warning::
 This means, if you have a high stack voltage and a low external voltage,
 the motor will be driven with the low external voltage. If you then remove
 the external connection, it will immediately be driven by the high
 stack voltage
"""
function get_external_input_voltage(device::BrickStepper)
    return device.deviceInternal.get_external_input_voltage()
end

export get_current_consumption
"""
    $(SIGNATURES)

Returns the current consumption of the motor.
"""
function get_current_consumption(device::BrickStepper)
    return device.deviceInternal.get_current_consumption()
end

export set_motor_current
"""
    $(SIGNATURES)

Sets the current with which the motor will be driven.

.. warning::
 Do not set this value above the specifications of your stepper motor.
 Otherwise it may damage your motor.
"""
function set_motor_current(device::BrickStepper, current)
    device.deviceInternal.set_motor_current(current)
end

export get_motor_current
"""
    $(SIGNATURES)

Returns the current as set by :func:`Set Motor Current`.
"""
function get_motor_current(device::BrickStepper)
    return device.deviceInternal.get_motor_current()
end

export enable
"""
    $(SIGNATURES)

Enables the driver chip. The driver parameters can be configured (maximum velocity,
acceleration, etc) before it is enabled.
"""
function enable(device::BrickStepper)
    device.deviceInternal.enable()
end

export disable
"""
    $(SIGNATURES)

Disables the driver chip. The configurations are kept (maximum velocity,
acceleration, etc) but the motor is not driven until it is enabled again.

.. warning::
 Disabling the driver chip while the motor is still turning can damage the
 driver chip. The motor should be stopped calling :func:`Stop` function
 before disabling the motor power. The :func:`Stop` function will **not**
 wait until the motor is actually stopped. You have to explicitly wait for the
 appropriate time after calling the :func:`Stop` function before calling
 the :func:`Disable` function.
"""
function disable(device::BrickStepper)
    device.deviceInternal.disable()
end

export is_enabled
"""
    $(SIGNATURES)

Returns *true* if the driver chip is enabled, *false* otherwise.
"""
function is_enabled(device::BrickStepper)
    return device.deviceInternal.is_enabled()
end

export set_decay
"""
    $(SIGNATURES)

Sets the decay mode of the stepper motor.
A value of 0 sets the fast decay mode, a value of
65535 sets the slow decay mode and a value in between sets the mixed
decay mode.

Changing the decay mode is only possible if synchronous rectification
is enabled (see :func:`Set Sync Rect`).

For a good explanation of the different decay modes see
`this <https://ebldc.com/?p=86/>`__ blog post by Avayan.

A good decay mode is unfortunately different for every motor. The best
way to work out a good decay mode for your stepper motor, if you can't
measure the current with an oscilloscope, is to listen to the sound of
the motor. If the value is too low, you often hear a high pitched
sound and if it is too high you can often hear a humming sound.

Generally, fast decay mode (small value) will be noisier but also
allow higher motor speeds.

.. note::
 There is unfortunately no formula to calculate a perfect decay
 mode for a given stepper motor. If you have problems with loud noises
 or the maximum motor speed is too slow, you should try to tinker with
 the decay value
"""
function set_decay(device::BrickStepper, decay)
    device.deviceInternal.set_decay(decay)
end

export get_decay
"""
    $(SIGNATURES)

Returns the decay mode as set by :func:`Set Decay`.
"""
function get_decay(device::BrickStepper)
    return device.deviceInternal.get_decay()
end

export set_minimum_voltage
"""
    $(SIGNATURES)

Sets the minimum voltage, below which the :cb:`Under Voltage` callback
is triggered. The minimum possible value that works with the Stepper Brick is 8V.
You can use this function to detect the discharge of a battery that is used
to drive the stepper motor. If you have a fixed power supply, you likely do
not need this functionality.
"""
function set_minimum_voltage(device::BrickStepper, voltage)
    device.deviceInternal.set_minimum_voltage(voltage)
end

export get_minimum_voltage
"""
    $(SIGNATURES)

Returns the minimum voltage as set by :func:`Set Minimum Voltage`.
"""
function get_minimum_voltage(device::BrickStepper)
    return device.deviceInternal.get_minimum_voltage()
end

export set_sync_rect
"""
    $(SIGNATURES)

Turns synchronous rectification on or off (*true* or *false*).

With synchronous rectification on, the decay can be changed
(see :func:`Set Decay`). Without synchronous rectification fast
decay is used.

For an explanation of synchronous rectification see
`here <https://en.wikipedia.org/wiki/Active_rectification>`__.

.. warning::
 If you want to use high speeds (> 10000 steps/s) for a large
 stepper motor with a large inductivity we strongly
 suggest that you disable synchronous rectification. Otherwise the
 Brick may not be able to cope with the load and overheat.
"""
function set_sync_rect(device::BrickStepper, sync_rect)
    device.deviceInternal.set_sync_rect(sync_rect)
end

export is_sync_rect
"""
    $(SIGNATURES)

Returns *true* if synchronous rectification is enabled, *false* otherwise.
"""
function is_sync_rect(device::BrickStepper)
    return device.deviceInternal.is_sync_rect()
end

export set_time_base
"""
    $(SIGNATURES)

Sets the time base of the velocity and the acceleration of the stepper brick.

For example, if you want to make one step every 1.5 seconds, you can set
the time base to 15 and the velocity to 10. Now the velocity is
10steps/15s = 1steps/1.5s.
"""
function set_time_base(device::BrickStepper, time_base)
    device.deviceInternal.set_time_base(time_base)
end

export get_time_base
"""
    $(SIGNATURES)

Returns the time base as set by :func:`Set Time Base`.
"""
function get_time_base(device::BrickStepper)
    return device.deviceInternal.get_time_base()
end

export get_all_data
"""
    $(SIGNATURES)

Returns the following parameters: The current velocity,
the current position, the remaining steps, the stack voltage, the external
voltage and the current consumption of the stepper motor.

There is also a callback for this function, see :cb:`All Data` callback.
"""
function get_all_data(device::BrickStepper)
    return device.deviceInternal.get_all_data()
end

export set_all_data_period
"""
    $(SIGNATURES)

Sets the period with which the :cb:`All Data` callback is triggered
periodically. A value of 0 turns the callback off.
"""
function set_all_data_period(device::BrickStepper, period)
    device.deviceInternal.set_all_data_period(period)
end

export get_all_data_period
"""
    $(SIGNATURES)

Returns the period as set by :func:`Set All Data Period`.
"""
function get_all_data_period(device::BrickStepper)
    return device.deviceInternal.get_all_data_period()
end

export set_spitfp_baudrate_config
"""
    $(SIGNATURES)

The SPITF protocol can be used with a dynamic baudrate. If the dynamic baudrate is
enabled, the Brick will try to adapt the baudrate for the communication
between Bricks and Bricklets according to the amount of data that is transferred.

The baudrate will be increased exponentially if lots of data is sent/received and
decreased linearly if little data is sent/received.

This lowers the baudrate in applications where little data is transferred (e.g.
a weather station) and increases the robustness. If there is lots of data to transfer
(e.g. Thermal Imaging Bricklet) it automatically increases the baudrate as needed.

In cases where some data has to transferred as fast as possible every few seconds
(e.g. RS485 Bricklet with a high baudrate but small payload) you may want to turn
the dynamic baudrate off to get the highest possible performance.

The maximum value of the baudrate can be set per port with the function
:func:`Set SPITFP Baudrate`. If the dynamic baudrate is disabled, the baudrate
as set by :func:`Set SPITFP Baudrate` will be used statically.

.. versionadded:: 2.3.6\$nbsp;(Firmware)
"""
function set_spitfp_baudrate_config(device::BrickStepper, enable_dynamic_baudrate, minimum_dynamic_baudrate)
    device.deviceInternal.set_spitfp_baudrate_config(enable_dynamic_baudrate, minimum_dynamic_baudrate)
end

export get_spitfp_baudrate_config
"""
    $(SIGNATURES)

Returns the baudrate config, see :func:`Set SPITFP Baudrate Config`.

.. versionadded:: 2.3.6\$nbsp;(Firmware)
"""
function get_spitfp_baudrate_config(device::BrickStepper)
    return device.deviceInternal.get_spitfp_baudrate_config()
end

export get_send_timeout_count
"""
    $(SIGNATURES)

Returns the timeout count for the different communication methods.

The methods 0-2 are available for all Bricks, 3-7 only for Master Bricks.

This function is mostly used for debugging during development, in normal operation
the counters should nearly always stay at 0.

.. versionadded:: 2.3.4\$nbsp;(Firmware)
"""
function get_send_timeout_count(device::BrickStepper, communication_method)
    return device.deviceInternal.get_send_timeout_count(communication_method)
end

export set_spitfp_baudrate
"""
    $(SIGNATURES)

Sets the baudrate for a specific Bricklet port.

If you want to increase the throughput of Bricklets you can increase
the baudrate. If you get a high error count because of high
interference (see :func:`Get SPITFP Error Count`) you can decrease the
baudrate.

If the dynamic baudrate feature is enabled, the baudrate set by this
function corresponds to the maximum baudrate (see :func:`Set SPITFP Baudrate Config`).

Regulatory testing is done with the default baudrate. If CE compatibility
or similar is necessary in your applications we recommend to not change
the baudrate.

.. versionadded:: 2.3.3\$nbsp;(Firmware)
"""
function set_spitfp_baudrate(device::BrickStepper, bricklet_port, baudrate)
    device.deviceInternal.set_spitfp_baudrate(bricklet_port, baudrate)
end

export get_spitfp_baudrate
"""
    $(SIGNATURES)

Returns the baudrate for a given Bricklet port, see :func:`Set SPITFP Baudrate`.

.. versionadded:: 2.3.3\$nbsp;(Firmware)
"""
function get_spitfp_baudrate(device::BrickStepper, bricklet_port)
    return device.deviceInternal.get_spitfp_baudrate(bricklet_port)
end

export get_spitfp_error_count
"""
    $(SIGNATURES)

Returns the error count for the communication between Brick and Bricklet.

The errors are divided into

* ACK checksum errors,
* message checksum errors,
* framing errors and
* overflow errors.

The errors counts are for errors that occur on the Brick side. All
Bricklets have a similar function that returns the errors on the Bricklet side.

.. versionadded:: 2.3.3\$nbsp;(Firmware)
"""
function get_spitfp_error_count(device::BrickStepper, bricklet_port)
    return device.deviceInternal.get_spitfp_error_count(bricklet_port)
end

export enable_status_led
"""
    $(SIGNATURES)

Enables the status LED.

The status LED is the blue LED next to the USB connector. If enabled is is
on and it flickers if data is transfered. If disabled it is always off.

The default state is enabled.

.. versionadded:: 2.3.1\$nbsp;(Firmware)
"""
function enable_status_led(device::BrickStepper)
    device.deviceInternal.enable_status_led()
end

export disable_status_led
"""
    $(SIGNATURES)

Disables the status LED.

The status LED is the blue LED next to the USB connector. If enabled is is
on and it flickers if data is transfered. If disabled it is always off.

The default state is enabled.

.. versionadded:: 2.3.1\$nbsp;(Firmware)
"""
function disable_status_led(device::BrickStepper)
    device.deviceInternal.disable_status_led()
end

export is_status_led_enabled
"""
    $(SIGNATURES)

Returns *true* if the status LED is enabled, *false* otherwise.

.. versionadded:: 2.3.1\$nbsp;(Firmware)
"""
function is_status_led_enabled(device::BrickStepper)
    return device.deviceInternal.is_status_led_enabled()
end

export get_protocol1_bricklet_name
"""
    $(SIGNATURES)

Returns the firmware and protocol version and the name of the Bricklet for a
given port.

This functions sole purpose is to allow automatic flashing of v1.x.y Bricklet
plugins.
"""
function get_protocol1_bricklet_name(device::BrickStepper, port)
    return device.deviceInternal.get_protocol1_bricklet_name(port)
end

export get_chip_temperature
"""
    $(SIGNATURES)

Returns the temperature as measured inside the microcontroller. The
value returned is not the ambient temperature!

The temperature is only proportional to the real temperature and it has an
accuracy of ±15%. Practically it is only useful as an indicator for
temperature changes.
"""
function get_chip_temperature(device::BrickStepper)
    return device.deviceInternal.get_chip_temperature()
end

export reset
"""
    $(SIGNATURES)

Calling this function will reset the Brick. Calling this function
on a Brick inside of a stack will reset the whole stack.

After a reset you have to create new device objects,
calling functions on the existing ones will result in
undefined behavior!
"""
function reset(device::BrickStepper)
    device.deviceInternal.reset()
end

export write_bricklet_plugin
"""
    $(SIGNATURES)

Writes 32 bytes of firmware to the bricklet attached at the given port.
The bytes are written to the position offset * 32.

This function is used by Brick Viewer during flashing. It should not be
necessary to call it in a normal user program.
"""
function write_bricklet_plugin(device::BrickStepper, port, offset, chunk)
    device.deviceInternal.write_bricklet_plugin(port, offset, chunk)
end

export read_bricklet_plugin
"""
    $(SIGNATURES)

Reads 32 bytes of firmware from the bricklet attached at the given port.
The bytes are read starting at the position offset * 32.

This function is used by Brick Viewer during flashing. It should not be
necessary to call it in a normal user program.
"""
function read_bricklet_plugin(device::BrickStepper, port, offset)
    return device.deviceInternal.read_bricklet_plugin(port, offset)
end

export get_identity
"""
    $(SIGNATURES)

Returns the UID, the UID where the Brick is connected to,
the position, the hardware and firmware version as well as the
device identifier.

The position is the position in the stack from '0' (bottom) to '8' (top).

The device identifier numbers can be found :ref:`here <device_identifier>`.
|device_identifier_constant|
"""
function get_identity(device::BrickStepper)
    return device.deviceInternal.get_identity()
end

export register_callback
"""
Registers the given *function* with the given *callback_id*.
"""
function register_callback(device::BrickStepper, callback_id, function_)
    if isnothing(function_)
        device.registered_callbacks.pop(callback_id, None)
    else
        device.registered_callbacks[callback_id] = function_
    end
end


# extracts vocabularies and emojis per region
# run as:
# JULIA_PROJECT=. julia -p64 -Lsrc/vocabulary-and-emojis.jl
# and then main(lang)

using CSV, Glob, DataFrames, JLD2, CodecZlib, TextSearch, InvertedFiles
using InvertedFiles: add!

include("io.jl")

function compute_vocabulary(filename, mindocs=5)
    voc = Dict{String,Int}()
    sizehint!(voc, 100_000)
    @info "computing vocabulary for $filename"

    n = 0
    doc = Set{String}()
    for t in eachline(filename)
        n += 1
        empty!(doc)

        for w in split(t)
            if !(w in doc)
                voc[w] = get(voc, w, 0) + 1
                push!(doc, w)
            end
        end
    end

    m = length(voc)

    for (k, freq) in voc
        if freq < mindocs
            delete!(voc, k)
        end
    end

    @info "computed vocabulary for $filename, n=$n, m=$m, final voc: $(length(voc))"
    voc, m, n, filename
end

function vocabulary_table(lang)
    vocfile = "data/$lang/voc.tsv.gz"
    vocstatsfile = "data/$lang/voc-stats.tsv.gz"
    if isfile(vocfile) && isfile(vocstatsfile)
        @info "vocabulary files already exists"
        return
    end
    VOC = DataFrame(token=String[], country_code=String[], ndocs=Int[], idf=Float64[])
    STATS = DataFrame(country_code=String[], n=Int[], rawvoc=Int[], voc=Int[])
    ALL = Dict{String,Int}()
    N = 0

    L = pmap(compute_vocabulary, glob("data/$lang/messages/*.txt"))

    for (voc, m, n, filename) in L
        cc = basename(filename)[1:2]
        N += n
        push!(STATS, (cc, n, m, length(voc)))
        for (token, ndocs) in voc
            prob = (ndocs + 1) / n
            push!(VOC, (token, cc, ndocs, log(1 / prob)))
            ALL[token] = get(ALL, token, 0) + ndocs
        end
    end

    for (token, ndocs) in ALL
        prob = (ndocs + 1) / N
        push!(VOC, (token, "ALL", ndocs, log(1 / prob)))
    end

    M = length(ALL)
    push!(STATS, ("ALL", N, M, M))
    sort!(VOC, :token)

    @info "saving voc files"
    open(vocfile, "w") do f
        gz = GzipCompressorStream(f)
        CSV.write(gz, VOC, delim='\t')
        close(gz)
    end

    open(vocstatsfile, "w") do f
        gz = GzipCompressorStream(f)
        CSV.write(gz, STATS, delim='\t')
        close(gz)
    end
end

function count_emojis(filename)
    voc = load(filename, "voc")

    D = Dict{Char,Int}()
    for (token, freq) in voc
        # there are some cases when the tokenizer considers emojis as part of a word, e.g.,
        # when they start the word
        lst = emojis(token)
        if length(lst) > 0
            for e in lst
                D[e] = get(D, e, 0) + row.ndocs
            end
        end
    end

    replace(basename(filename), "voc-" => "", ".jld2" => "") => D
end

function emoji_table(lang)
    outname = "data/$lang/emojis.tsv.gz"
    isfile(outname) && return

    V = CSV.read("data/$lang/voc.tsv.gz", DataFrame, delim='\t')
    E = DataFrame(emoji=Char[], country_code=[], freq=Int[])

    D = Dict()

    for row in eachrow(V)
        cc = row.country_code

        for e in emojis(row.token)
            D[(e, cc)] = get(D, (e, cc), 0) + row.ndocs
        end
    end

    for (k, freq) in D
        e, cc = k
        push!(E, (e, cc, freq))
    end

    sort!(E, :emoji)

    open(outname, "w") do f
        gz = GzipCompressorStream(f)
        CSV.write(gz, E, delim='\t')
        close(gz)
    end
end

function normalize_text_from_csv(filename)
    config = TextConfig(lc=true, del_diac=true, del_dup=true, del_punc=false, group_num=true)
    tok = Tokenizer(config)
    @info filename
    D = CSV.read(filename, DataFrame, delim='\t')
    T = Vector{String}(undef, length(D.text))

    for (i, t) in enumerate(D.text)
        tokens = tokenize(tok, t)
        T[i] = join(decode.(tok, tokens), ' ')
    end

    T
end

function extract_text(ccpath)
    outname = rstrip(ccpath, '/') * ".txt"
    isfile(outname) && return
    L = pmap(normalize_text_from_csv, glob(joinpath(ccpath, "*.tsv.gz")))

    open(outname * ".tmp", "w") do f
        for lst in L
            for t in lst
                println(f, t)
            end
        end
    end

    mv(outname * ".tmp", outname; force=true)
end

function extract_id(ccpath)
    outname = rstrip(ccpath, '/') * "-id-list.txt.gz"
    isfile(outname) && return

    open(outname * ".tmp", "w") do f
        gz = GzipCompressorStream(f)
        for filename in glob(joinpath(ccpath, "*.tsv.gz"))
            @info "extracting identifiers from $filename"
            D = CSV.read(filename, DataFrame, delim='\t')
            for id in D.id
                println(gz, id)
            end
        end

        close(gz)
    end

    mv(outname * ".tmp", outname; force=true)
end

function main(lang)

    for ccpath in glob("data/$lang/messages/*/")
        extract_text(ccpath)
    end

    @sync @distributed for ccpath in glob("data/$lang/messages/*/")
        extract_id(ccpath)
    end

    vocabulary_table(lang)
    emoji_table(lang)
end


function _precompile_()
    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing
    Base.precompile(Tuple{typeof(ROT),GVar{GVar{Float64, Float64}, GVar{Float64, Float64}},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.08222882
    Base.precompile(Tuple{MinusEq{typeof(complex)},GVar{Float64, Float64},Union{GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}},Union{GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}}})   # time: 0.06912181
    Base.precompile(Tuple{typeof(wrap_bcastgrad),Type{AutoBcast{Int64, 4}},Vector{Float64}})   # time: 0.05026503
    Base.precompile(Tuple{Type{GVar},Matrix{Float64}})   # time: 0.044000912
    Base.precompile(Tuple{MinusEq{typeof(sincos)},Tuple{GVar{Float64, Float64}, GVar{Float64, Float64}},GVar{Float64, Float64}})   # time: 0.042384867
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, Float64},Any})   # time: 0.03920624
    Base.precompile(Tuple{typeof(NiLangCore.loaddata),Vector{Any},Vector{Any}})   # time: 0.03768901
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(-)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.036892496
    Base.precompile(Tuple{typeof(NiLangCore.loaddata),Type{Vector{Float64}},Vector{Float64}})   # time: 0.035732098
    Base.precompile(Tuple{MinusEq{typeof(log)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.03535623
    Base.precompile(Tuple{PlusEq{typeof(complex)},GVar{Float64, Float64},Union{GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}},Union{GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}}})   # time: 0.03526533
    Base.precompile(Tuple{typeof(grad),NTuple{4, GVar{Float64, Float64}}})   # time: 0.032369606
    Base.precompile(Tuple{typeof(wrap_jacobian),Type{AutoBcast{Float64, 1}},Vector{Float64}})   # time: 0.032170866
    Base.precompile(Tuple{Type{GVar},Vector{Float64}})   # time: 0.029704165
    Base.precompile(Tuple{typeof(IROT),GVar{Float64, Float64},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.025739137
    Base.precompile(Tuple{PlusEq{typeof(^)},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.024799693
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.023590498
    Base.precompile(Tuple{typeof(grad),Vector{GVar{Float64, Float64}}})   # time: 0.02317364
    Base.precompile(Tuple{typeof(grad),Tuple{Vector{GVar{Float64, Float64}}, Vector{GVar{Float64, Float64}}, Vector{GVar{Float64, Float64}}, GVar{Float64, Float64}}})   # time: 0.022934826
    Base.precompile(Tuple{PlusEq{typeof(^)},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}},GVar{GVar{Float64, Float64}, GVar{Float64, Float64}},Int64})   # time: 0.022084313
    Base.precompile(Tuple{typeof(NiLangCore.loaddata),Vector{GVar{Float64, Float64}},Vector{Float64}})   # time: 0.02116515
    Base.precompile(Tuple{MinusEq{typeof(/)},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.020155573
    Base.precompile(Tuple{PlusEq{typeof(*)},GVar{Float64, Float64},Any,Float64})   # time: 0.016616046
    Base.precompile(Tuple{typeof(IROT),GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.016194412
    Base.precompile(Tuple{Type{GVar},Tuple{Float64, ComplexF64},Tuple{ComplexF64, ComplexF64}})   # time: 0.01619109
    Base.precompile(Tuple{PlusEq{typeof(^)},GVar{Float64, Float64},Any,Int64})   # time: 0.015741551
    Base.precompile(Tuple{Type{GVar},AbstractArray,AbstractArray})   # time: 0.013830282
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(inv)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.011192932
    Base.precompile(Tuple{typeof(wrap_jacobian),Type{AutoBcast{Float64, 3}},Vector{Float64}})   # time: 0.011075067
    Base.precompile(Tuple{typeof(wrap_jacobian),Type{AutoBcast{Float64, 2}},Vector{Float64}})   # time: 0.011053615
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(^)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, GVar{Float64, Float64}}})   # time: 0.010716249
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(complex)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.010689858
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(/)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.010517918
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(*)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.01049392
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(-)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.010452602
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(*)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.01041573
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(^)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, GVar{Float64, Float64}}})   # time: 0.010397522
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(+)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.010353689
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(complex)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.010343891
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(/)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.010335974
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(exp)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.010250594
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(+)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.010101405
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(log)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.009795744
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(identity)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.009747697
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(exp)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.009696897
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(^)}, Union{GVar{Float64, Float64}, Float64}, Vararg{Any, N} where N}})   # time: 0.009630851
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(inv)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.009559213
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(log)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.009503618
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.009323083
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(identity)}, GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}, Complex{GVar{Float64, Float64}}}})   # time: 0.0093146
    Base.precompile(Tuple{MinusEq{typeof(cos)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.009097533
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(^)}, Union{GVar{Float64, Float64}, Float64}, Vararg{Any, N} where N}})   # time: 0.009068273
    Base.precompile(Tuple{typeof(grad),Tuple{MinusEq{typeof(complex)}, Union{GVar{Float64, Float64}, Float64}, Complex{GVar{Float64, Float64}}, GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.008931454
    Base.precompile(Tuple{PlusEq{typeof(^)},GVar{Float64, Float64},GVar,Int64})   # time: 0.008520359
    Base.precompile(Tuple{typeof(grad),Tuple{PlusEq{typeof(complex)}, Union{GVar{Float64, Float64}, Float64}, Complex{GVar{Float64, Float64}}, GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.00843281
    Base.precompile(Tuple{typeof(wrap_bcastgrad),Type{AutoBcast{Int64, 4}},Tuple{Float64}})   # time: 0.007901799
    Base.precompile(Tuple{typeof(grad),Tuple{typeof(NEG), GVar{Float64, Float64}, Complex{GVar{Float64, Float64}}}})   # time: 0.007657704
    Base.precompile(Tuple{PlusEq{typeof(/)},GVar{Float64, Float64},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.007580364
    Base.precompile(Tuple{typeof(grad),Tuple{GVar{Float64, Float64}, Matrix{GVar{Float64, Float64}}}})   # time: 0.007370161
    Base.precompile(Tuple{typeof(grad),Tuple{GVar{Float64, Float64}, GVar{Float64, Float64}, Vector{GVar{Float64, Float64}}}})   # time: 0.007349056
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, Float64},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.007118214
    Base.precompile(Tuple{PlusEq{typeof(identity)},Vector{GVar{Float64, Float64}},Vector{GVar{Float64, Float64}}})   # time: 0.007062924
    Base.precompile(Tuple{MinusEq{typeof(^)},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.007059243
    Base.precompile(Tuple{typeof(grad),Tuple{Float64, GVar{Float64, Float64}}})   # time: 0.006490685
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.005542749
    Base.precompile(Tuple{MinusEq{typeof(exp)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.004707843
    Base.precompile(Tuple{typeof(zero),GVar{Int64, AutoBcast{Int64, 3}}})   # time: 0.004680167
    Base.precompile(Tuple{typeof(grad),Tuple{GVar{Float64, Float64}, GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.00466041
    Base.precompile(Tuple{MinusEq{typeof(abs)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.004496061
    Base.precompile(Tuple{MinusEq{typeof(log)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.003723256
    Base.precompile(Tuple{PlusEq{typeof(/)},GVar{Float64, Float64},GVar,GVar{Float64, Float64}})   # time: 0.003611694
    Base.precompile(Tuple{typeof(grad),Tuple{GVar{Float64, Float64}, GVar{Float64, Float64}}})   # time: 0.003529446
    Base.precompile(Tuple{MinusEq{typeof(/)},GVar{Float64, Float64},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.003528577
    Base.precompile(Tuple{MinusEq{typeof(tan)},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.003438413
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, Float64},GVar})   # time: 0.003361321
    Base.precompile(Tuple{typeof(grad),Tuple{Float64, Float64}})   # time: 0.003360894
    Base.precompile(Tuple{Type{Inv{GVar}},AbstractArray})   # time: 0.003287998
    Base.precompile(Tuple{typeof(SWAP),GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.003202025
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.003030533
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, AutoBcast{Float64, 3}},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.002981777
    Base.precompile(Tuple{MinusEq{typeof(/)},GVar{Float64, Float64},GVar{Float64, Float64},Int64})   # time: 0.002951113
    Base.precompile(Tuple{MinusEq{typeof(log)},GVar{Float64, Float64},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.002914175
    Base.precompile(Tuple{Type{GVar},Float64,AutoBcast{Float64, _A} where _A})   # time: 0.002864071
    Base.precompile(Tuple{PlusEq{typeof(*)},GVar{Float64, Float64},GVar,Float64})   # time: 0.00271298
    Base.precompile(Tuple{PlusEq{typeof(/)},GVar{Float64, Float64},Real,GVar{Float64, Float64}})   # time: 0.002609891
    Base.precompile(Tuple{typeof(NiLangCore.loaddata),Type{GVar{Float64, AutoBcast{Float64, 3}}},Float64})   # time: 0.002460321
    Base.precompile(Tuple{typeof(-),GVar{Float64, AutoBcast{Float64, 1}}})   # time: 0.002408763
    Base.precompile(Tuple{typeof(NEG),GVar{Float64, AutoBcast{Float64, 1}}})   # time: 0.00236964
    Base.precompile(Tuple{PlusEq{typeof(^)},GVar{Float64, Float64},Float64,Int64})   # time: 0.002257135
    Base.precompile(Tuple{PlusEq{typeof(identity)},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.002224634
    Base.precompile(Tuple{PlusEq{typeof(*)},GVar{Float64, Float64},Real,Float64})   # time: 0.002007978
    Base.precompile(Tuple{SubConst{Float64},GVar{Float64, Float64}})   # time: 0.001776433
    Base.precompile(Tuple{PlusEq{typeof(^)},GVar{Float64, Float64},Real,Int64})   # time: 0.00169386
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, Float64},Float64,GVar{Float64, Float64}})   # time: 0.001672073
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, Float64},Int64,GVar{Float64, Float64}})   # time: 0.001642821
    Base.precompile(Tuple{PlusEq{typeof(*)},GVar{Float64, Float64},Float64,GVar{Float64, Float64}})   # time: 0.001626397
    Base.precompile(Tuple{AddConst{Float64},GVar{Float64, Float64}})   # time: 0.001618372
    Base.precompile(Tuple{PlusEq{typeof(*)},GVar{Float64, Float64},Int64,GVar{Float64, Float64}})   # time: 0.001491792
    Base.precompile(Tuple{PlusEq{typeof(*)},GVar{Float64, Float64},GVar{Float64, Float64},Float64})   # time: 0.001471461
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, Float64},GVar{Float64, Float64},Float64})   # time: 0.001428104
    Base.precompile(Tuple{Type{GVar},ComplexF64})   # time: 0.001348701
    Base.precompile(Tuple{MinusEq{typeof(cos)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001196095
    Base.precompile(Tuple{MinusEq{typeof(sin)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001177611
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001173775
    Base.precompile(Tuple{MinusEq{typeof(abs)},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001170435
    Base.precompile(Tuple{typeof(chfield),Complex{GVar{Float64, Float64}},typeof(grad),ComplexF64})   # time: 0.00116481
    Base.precompile(Tuple{MinusEq{typeof(abs)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001158102
    Base.precompile(Tuple{MinusEq{typeof(exp)},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001156032
    Base.precompile(Tuple{MinusEq{typeof(*)},GVar{Float64, Float64},GVar{Float64, Float64}})   # time: 0.001134847
    Base.precompile(Tuple{MinusEq{typeof(sin)},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001125385
    Base.precompile(Tuple{MinusEq{typeof(cos)},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001105154
    Base.precompile(Tuple{MinusEq{typeof(exp)},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001097152
    Base.precompile(Tuple{MinusEq{typeof(log)},GVar{Float64, AutoBcast{Float64, 2}}})   # time: 0.001096638
    Base.precompile(Tuple{MinusEq{typeof(/)},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.001095843
    Base.precompile(Tuple{MinusEq{typeof(^)},GVar{Float64, AutoBcast{Float64, 3}}})   # time: 0.001095656
    Base.precompile(Tuple{typeof(chfield),GVar{Float64, Float64},typeof(grad),Float64})   # time: 0.001003963
    Base.precompile(Tuple{MinusEq{typeof(identity)},GVar{Float64, Float64},Real})   # time: 0.00100021
end


module SrtmKgb25
  # KURUCZ
  sfluxref = [42.6858 , 45.7720, 44.9872, 45.9662    ,46.5458 , 41.6926, 32.2893, 24.0928    ,16.7686 , 1.86048, 1.54057, 1.23503    ,0.915085,0.590099,0.218622, 3.21287e-02]

  #     Rayleigh extinction coefficient at v = 2925 cm-1.
  rayl = [9.81132e-07,8.25605e-07,6.71302e-07,5.53556e-07,
    3.97383e-07,3.68206e-07,4.42379e-07,4.57799e-07,4.22683e-07,3.87113e-07,3.79810e-07,3.63192e-07,3.51921e-07,3.34231e-07,3.34294e-07,3.32673e-07]

  abso3a = [2.32664e-02,5.76154e-02,0.125389,0.250158,0.378756   ,0.402196   ,0.352026,0.352036,0.386253   ,0.414598   ,0.420079,0.435471,0.445487   ,0.459549   ,0.452920,0.456838]

  abso3b = [
    1.76917e-02,4.64185e-02,1.03640e-01,0.189469,0.303858   ,0.400248   ,0.447357   ,0.470009,0.498673   ,0.515696   ,0.517053   ,0.517930,0.518345   ,0.524952   ,0.508244   ,0.468981]

  layreffr = 2

  #     ------------------------------------------------------------------

  #     The array KA contains absorption coefs at the 16 chosen g-values
  #     for a range of pressure levels> ~100mb:and binary:temperatures
  #     species parameters (see taumol.f for definition).  The first
  #     index in the array, JS, runs from 1 to 9, and corresponds to
  #     different values of the binary species parameter.  For instance,
  #     JS=1 refers to dry air, JS = 2 corresponds to the paramter value 1/8,
  #     JS = 3 corresponds to the parameter value 2/8, etc.  The second index
  #     in the array, JT, which runs from 1 to 5, corresponds to different
  #     temperatures.  More specifically, JT = 3 means that the data are for
  #     the reference temperature TREF for this  pressure level:JT = 2 refers
  #     to TREF-15, JT = 1 is for TREF-30:and JT = 5:JT = 4 is for TREF+15
  #     is for TREF+30.  The third index:runs from 1 to 13 and refers:JP
  #     to the JPth reference pressure level (see taumol.f for these levels
  #     in mb).  The fourth index, IG, goes from 1 to 16, and indicates
  #     which g-interval the absorption coefficients are for.
  #     -----------------------------------------------------------------
  ka = zeros(5,13,16)
  ka[:, 1, 1] = [0.16461e-08,0.16782e-08,0.19339e-08,0.17100e-08,0.17045e-08]
  ka[:, 2, 1] = [0.28759e-08,0.29469e-08,0.33789e-08,0.34357e-08,0.28833e-08]
  ka[:, 3, 1] = [0.55148e-08,0.54808e-08,0.54190e-08,0.68260e-08,0.51972e-08]
  ka[:, 4, 1] = [0.95336e-08,0.94552e-08,0.93001e-08,0.90961e-08,0.14451e-07]
  ka[:, 5, 1] = [0.14930e-07,0.14736e-07,0.14432e-07,0.14074e-07,0.24102e-07]
  ka[:, 6, 1] = [0.22770e-07,0.22301e-07,0.21778e-07,0.21194e-07,0.20569e-07]
  ka[:, 7, 1] = [0.34699e-07,0.33951e-07,0.33124e-07,0.32144e-07,0.31220e-07]
  ka[:, 8, 1] = [0.62339e-07,0.60405e-07,0.59548e-07,0.58214e-07,0.56977e-07]
  ka[:, 9, 1] = [0.17411e-06,0.17654e-06,0.18315e-06,0.18100e-06,0.17839e-06]
  ka[:,10, 1] = [0.23526e-06,0.22729e-06,0.21947e-06,0.21188e-06,0.20454e-06]
  ka[:,11, 1] = [0.23535e-06,0.22737e-06,0.21956e-06,0.21196e-06,0.20461e-06]
  ka[:,12, 1] = [0.23539e-06,0.22740e-06,0.21959e-06,0.21199e-06,0.20465e-06]
  ka[:,13, 1] = [0.23543e-06,0.22744e-06,0.21962e-06,0.21202e-06,0.20467e-06]
  ka[:, 1, 2] = [0.62912e-08,0.61559e-08,0.84640e-08,0.59240e-08,0.58217e-08]
  ka[:, 2, 2] = [0.83749e-08,0.80756e-08,0.11623e-07,0.11272e-07,0.73636e-08]
  ka[:, 3, 2] = [0.13304e-07,0.12795e-07,0.12343e-07,0.21235e-07,0.11577e-07]
  ka[:, 4, 2] = [0.20704e-07,0.19736e-07,0.18900e-07,0.18228e-07,0.31601e-07]
  ka[:, 5, 2] = [0.31149e-07,0.29669e-07,0.28318e-07,0.27101e-07,0.49649e-07]
  ka[:, 6, 2] = [0.45713e-07,0.43519e-07,0.41488e-07,0.39918e-07,0.38291e-07]
  ka[:, 7, 2] = [0.77265e-07,0.73848e-07,0.70437e-07,0.67945e-07,0.66127e-07]
  ka[:, 8, 2] = [0.15754e-06,0.15664e-06,0.15378e-06,0.15027e-06,0.14633e-06]
  ka[:, 9, 2] = [0.16439e-06,0.14678e-06,0.12610e-06,0.11532e-06,0.10591e-06]
  ka[:,10, 2] = [0.14366e-06,0.13506e-06,0.12583e-06,0.11774e-06,0.11011e-06]
  ka[:,11, 2] = [0.14521e-06,0.13766e-06,0.13072e-06,0.12218e-06,0.11400e-06]
  ka[:,12, 2] = [0.14524e-06,0.13769e-06,0.13074e-06,0.12241e-06,0.11552e-06]
  ka[:,13, 2] = [0.14525e-06,0.13770e-06,0.13075e-06,0.12252e-06,0.11553e-06]
  ka[:, 1, 3] = [0.14060e-07,0.13587e-07,0.24644e-07,0.12716e-07,0.12367e-07]
  ka[:, 2, 3] = [0.17055e-07,0.16577e-07,0.32443e-07,0.31273e-07,0.15381e-07]
  ka[:, 3, 3] = [0.25414e-07,0.24672e-07,0.23874e-07,0.47281e-07,0.22346e-07]
  ka[:, 4, 3] = [0.39536e-07,0.38124e-07,0.36836e-07,0.35587e-07,0.72260e-07]
  ka[:, 5, 3] = [0.59488e-07,0.57630e-07,0.55623e-07,0.53878e-07,0.11230e-06]
  ka[:, 6, 3] = [0.99996e-07,0.96206e-07,0.93184e-07,0.90812e-07,0.89206e-07]
  ka[:, 7, 3] = [0.17678e-06,0.17554e-06,0.17358e-06,0.17091e-06,0.16830e-06]
  ka[:, 8, 3] = [0.18672e-06,0.17850e-06,0.16967e-06,0.16275e-06,0.15875e-06]
  ka[:, 9, 3] = [0.13558e-06,0.13493e-06,0.13633e-06,0.13799e-06,0.13932e-06]
  ka[:,10, 3] = [0.18883e-06,0.20452e-06,0.22206e-06,0.24347e-06,0.26091e-06]
  ka[:,11, 3] = [0.21296e-06,0.23580e-06,0.26439e-06,0.30148e-06,0.34942e-06]
  ka[:,12, 3] = [0.22072e-06,0.25535e-06,0.28661e-06,0.34814e-06,0.39337e-06]
  ka[:,13, 3] = [0.22515e-06,0.26161e-06,0.30833e-06,0.36527e-06,0.40123e-06]
  ka[:, 1, 4] = [0.32735e-07,0.31345e-07,0.58846e-07,0.28258e-07,0.27022e-07]
  ka[:, 2, 4] = [0.37754e-07,0.36873e-07,0.82776e-07,0.80947e-07,0.35190e-07]
  ka[:, 3, 4] = [0.76368e-07,0.75292e-07,0.74075e-07,0.10820e-06,0.73183e-07]
  ka[:, 4, 4] = [0.16392e-06,0.16130e-06,0.15926e-06,0.15700e-06,0.22041e-06]
  ka[:, 5, 4] = [0.29704e-06,0.28924e-06,0.28301e-06,0.27633e-06,0.42284e-06]
  ka[:, 6, 4] = [0.48466e-06,0.47240e-06,0.46143e-06,0.45012e-06,0.43867e-06]
  ka[:, 7, 4] = [0.71637e-06,0.69847e-06,0.67384e-06,0.65368e-06,0.63375e-06]
  ka[:, 8, 4] = [0.11904e-05,0.11714e-05,0.11524e-05,0.11354e-05,0.11172e-05]
  ka[:, 9, 4] = [0.21976e-05,0.21606e-05,0.21332e-05,0.20944e-05,0.20536e-05]
  ka[:,10, 4] = [0.21713e-05,0.21144e-05,0.20553e-05,0.19901e-05,0.19286e-05]
  ka[:,11, 4] = [0.21443e-05,0.20785e-05,0.20048e-05,0.19232e-05,0.18295e-05]
  ka[:,12, 4] = [0.21363e-05,0.20578e-05,0.19811e-05,0.18729e-05,0.17807e-05]
  ka[:,13, 4] = [0.21319e-05,0.20513e-05,0.19580e-05,0.18546e-05,0.17725e-05]
  ka[:, 1, 5] = [0.36050e-07,0.36125e-07,0.46253e-07,0.37280e-07,0.37359e-07]
  ka[:, 2, 5] = [0.65102e-07,0.64266e-07,0.68896e-07,0.65925e-07,0.61190e-07]
  ka[:, 3, 5] = [0.12173e-06,0.11889e-06,0.11625e-06,0.17574e-06,0.10921e-06]
  ka[:, 4, 5] = [0.20555e-06,0.19853e-06,0.19068e-06,0.18313e-06,0.30241e-06]
  ka[:, 5, 5] = [0.30900e-06,0.29996e-06,0.28857e-06,0.27772e-06,0.51631e-06]
  ka[:, 6, 5] = [0.43774e-06,0.42465e-06,0.40920e-06,0.39315e-06,0.37901e-06]
  ka[:, 7, 5] = [0.63869e-06,0.61654e-06,0.60324e-06,0.58966e-06,0.57948e-06]
  ka[:, 8, 5] = [0.98362e-06,0.96271e-06,0.94180e-06,0.92206e-06,0.91105e-06]
  ka[:, 9, 5] = [0.12061e-05,0.11895e-05,0.11564e-05,0.11296e-05,0.11110e-05]
  ka[:,10, 5] = [0.12958e-05,0.12694e-05,0.12425e-05,0.12153e-05,0.11880e-05]
  ka[:,11, 5] = [0.12962e-05,0.12698e-05,0.12429e-05,0.12156e-05,0.11883e-05]
  ka[:,12, 5] = [0.12964e-05,0.12701e-05,0.12431e-05,0.12158e-05,0.11885e-05]
  ka[:,13, 5] = [0.12966e-05,0.12702e-05,0.12433e-05,0.12160e-05,0.11886e-05]
  ka[:, 1, 6] = [0.73925e-07,0.70231e-07,0.21454e-06,0.63477e-07,0.60912e-07]
  ka[:, 2, 6] = [0.67794e-07,0.65807e-07,0.13854e-06,0.13061e-06,0.59361e-07]
  ka[:, 3, 6] = [0.98353e-07,0.95275e-07,0.92426e-07,0.15768e-06,0.87986e-07]
  ka[:, 4, 6] = [0.15855e-06,0.15394e-06,0.14948e-06,0.14655e-06,0.23172e-06]
  ka[:, 5, 6] = [0.27764e-06,0.26941e-06,0.26299e-06,0.25975e-06,0.40526e-06]
  ka[:, 6, 6] = [0.45469e-06,0.44417e-06,0.43276e-06,0.42440e-06,0.41489e-06]
  ka[:, 7, 6] = [0.71540e-06,0.71291e-06,0.70656e-06,0.69823e-06,0.68342e-06]
  ka[:, 8, 6] = [0.79651e-06,0.79807e-06,0.80621e-06,0.80941e-06,0.79835e-06]
  ka[:, 9, 6] = [0.18716e-06,0.16713e-06,0.14725e-06,0.13728e-06,0.11763e-06]
  ka[:,10, 6] = [0.92638e-07,0.86207e-07,0.80877e-07,0.70432e-07,0.64517e-07]
  ka[:,11, 6] = [0.13396e-06,0.12820e-06,0.12387e-06,0.10427e-06,0.94091e-07]
  ka[:,12, 6] = [0.14877e-06,0.14827e-06,0.14350e-06,0.12154e-06,0.10552e-06]
  ka[:,13, 6] = [0.15437e-06,0.15323e-06,0.14992e-06,0.12715e-06,0.10933e-06]
  ka[:, 1, 7] = [0.72717e-06,0.70656e-06,0.13933e-05,0.66449e-06,0.64269e-06]
  ka[:, 2, 7] = [0.52595e-06,0.50791e-06,0.11171e-05,0.10538e-05,0.45644e-06]
  ka[:, 3, 7] = [0.29919e-06,0.29227e-06,0.28284e-06,0.65215e-06,0.26347e-06]
  ka[:, 4, 7] = [0.27961e-06,0.27579e-06,0.27068e-06,0.26343e-06,0.41265e-06]
  ka[:, 5, 7] = [0.37031e-06,0.36318e-06,0.35475e-06,0.34488e-06,0.53740e-06]
  ka[:, 6, 7] = [0.53195e-06,0.52692e-06,0.52224e-06,0.51934e-06,0.51146e-06]
  ka[:, 7, 7] = [0.83043e-06,0.84552e-06,0.84833e-06,0.82800e-06,0.80930e-06]
  ka[:, 8, 7] = [0.14910e-05,0.15179e-05,0.15248e-05,0.15091e-05,0.14853e-05]
  ka[:, 9, 7] = [0.37340e-05,0.37823e-05,0.38311e-05,0.38453e-05,0.38567e-05]
  ka[:,10, 7] = [0.86791e-05,0.89697e-05,0.92118e-05,0.93991e-05,0.95564e-05]
  ka[:,11, 7] = [0.11878e-04,0.12201e-04,0.12588e-04,0.12897e-04,0.13151e-04]
  ka[:,12, 7] = [0.13192e-04,0.13732e-04,0.14137e-04,0.14465e-04,0.14643e-04]
  ka[:,13, 7] = [0.13716e-04,0.14229e-04,0.14617e-04,0.14944e-04,0.15182e-04]
  ka[:, 1, 8] = [0.39538e-05,0.38949e-05,0.56188e-05,0.37475e-05,0.36648e-05]
  ka[:, 2, 8] = [0.34231e-05,0.33633e-05,0.51877e-05,0.50048e-05,0.31425e-05]
  ka[:, 3, 8] = [0.28073e-05,0.27497e-05,0.26875e-05,0.44405e-05,0.25492e-05]
  ka[:, 4, 8] = [0.19229e-05,0.18818e-05,0.18382e-05,0.17896e-05,0.33073e-05]
  ka[:, 5, 8] = [0.11453e-05,0.11293e-05,0.11095e-05,0.10866e-05,0.19344e-05]
  ka[:, 6, 8] = [0.14565e-05,0.14517e-05,0.14369e-05,0.14141e-05,0.13944e-05]
  ka[:, 7, 8] = [0.23228e-05,0.22753e-05,0.22395e-05,0.22124e-05,0.21731e-05]
  ka[:, 8, 8] = [0.34877e-05,0.34362e-05,0.33796e-05,0.33389e-05,0.32924e-05]
  ka[:, 9, 8] = [0.63448e-05,0.63701e-05,0.63619e-05,0.62632e-05,0.61645e-05]
  ka[:,10, 8] = [0.12155e-04,0.11880e-04,0.11762e-04,0.11759e-04,0.11651e-04]
  ka[:,11, 8] = [0.14093e-04,0.13835e-04,0.13547e-04,0.13205e-04,0.12690e-04]
  ka[:,12, 8] = [0.14428e-04,0.14056e-04,0.13932e-04,0.13396e-04,0.12885e-04]
  ka[:,13, 8] = [0.15229e-04,0.14534e-04,0.13849e-04,0.13292e-04,0.12704e-04]
  ka[:, 1, 9] = [0.19250e-04,0.19148e-04,0.21702e-04,0.18906e-04,0.18761e-04]
  ka[:, 2, 9] = [0.18132e-04,0.18040e-04,0.20884e-04,0.20523e-04,0.17656e-04]
  ka[:, 3, 9] = [0.16928e-04,0.16843e-04,0.16742e-04,0.19715e-04,0.16470e-04]
  ka[:, 4, 9] = [0.15526e-04,0.15463e-04,0.15377e-04,0.15268e-04,0.18367e-04]
  ka[:, 5, 9] = [0.13545e-04,0.13511e-04,0.13455e-04,0.13362e-04,0.16722e-04]
  ka[:, 6, 9] = [0.97183e-05,0.97218e-05,0.97084e-05,0.96717e-05,0.96030e-05]
  ka[:, 7, 9] = [0.50307e-05,0.50984e-05,0.51628e-05,0.52093e-05,0.52354e-05]
  ka[:, 8, 9] = [0.45837e-05,0.45939e-05,0.45938e-05,0.45639e-05,0.45109e-05]
  ka[:, 9, 9] = [0.12254e-04,0.12319e-04,0.12397e-04,0.12584e-04,0.12620e-04]
  ka[:,10, 9] = [0.21545e-04,0.21836e-04,0.21718e-04,0.21511e-04,0.21211e-04]
  ka[:,11, 9] = [0.20079e-04,0.19539e-04,0.18859e-04,0.18393e-04,0.18181e-04]
  ka[:,12, 9] = [0.17115e-04,0.16357e-04,0.15410e-04,0.15220e-04,0.15207e-04]
  ka[:,13, 9] = [0.14935e-04,0.14679e-04,0.14593e-04,0.14448e-04,0.14436e-04]
  ka[:, 1,10] = [0.53569e-04,0.53042e-04,0.55454e-04,0.52098e-04,0.51678e-04]
  ka[:, 2,10] = [0.52196e-04,0.51739e-04,0.54777e-04,0.54075e-04,0.50624e-04]
  ka[:, 3,10] = [0.50339e-04,0.50046e-04,0.49769e-04,0.53168e-04,0.49370e-04]
  ka[:, 4,10] = [0.48505e-04,0.48316e-04,0.48143e-04,0.47993e-04,0.51621e-04]
  ka[:, 5,10] = [0.46313e-04,0.46267e-04,0.46119e-04,0.46064e-04,0.50279e-04]
  ka[:, 6,10] = [0.42662e-04,0.42818e-04,0.42935e-04,0.43007e-04,0.43099e-04]
  ka[:, 7,10] = [0.35762e-04,0.36149e-04,0.36450e-04,0.36639e-04,0.36887e-04]
  ka[:, 8,10] = [0.13516e-05,0.18607e-05,0.23061e-05,0.27339e-05,0.36516e-05]
  ka[:, 9,10] = [0.36432e-05,0.40739e-05,0.43830e-05,0.41136e-05,0.43128e-05]
  ka[:,10,10] = [0.62049e-05,0.69116e-05,0.73244e-05,0.65087e-05,0.78951e-05]
  ka[:,11,10] = [0.32156e-05,0.38834e-05,0.41231e-05,0.43386e-05,0.43405e-05]
  ka[:,12,10] = [0.22152e-05,0.26754e-05,0.31971e-05,0.34911e-05,0.37935e-05]
  ka[:,13,10] = [0.19792e-05,0.26543e-05,0.31511e-05,0.34597e-05,0.40624e-05]
  ka[:, 1,11] = [0.75384e-04,0.75103e-04,0.77406e-04,0.74222e-04,0.73734e-04]
  ka[:, 2,11] = [0.75458e-04,0.75244e-04,0.77778e-04,0.77018e-04,0.73942e-04]
  ka[:, 3,11] = [0.75023e-04,0.74844e-04,0.74477e-04,0.77271e-04,0.73633e-04]
  ka[:, 4,11] = [0.73633e-04,0.73539e-04,0.73257e-04,0.72934e-04,0.76232e-04]
  ka[:, 5,11] = [0.71348e-04,0.71322e-04,0.71227e-04,0.71069e-04,0.75258e-04]
  ka[:, 6,11] = [0.67784e-04,0.67873e-04,0.67974e-04,0.67924e-04,0.67903e-04]
  ka[:, 7,11] = [0.61855e-04,0.61922e-04,0.61973e-04,0.62206e-04,0.62496e-04]
  ka[:, 8,11] = [0.36622e-04,0.37413e-04,0.38740e-04,0.40550e-04,0.41833e-04]
  ka[:, 9,11] = [0.28544e-05,0.28831e-05,0.31445e-05,0.32900e-05,0.27967e-05]
  ka[:,10,11] = [0.53755e-05,0.42123e-05,0.51154e-05,0.63481e-05,0.54219e-05]
  ka[:,11,11] = [0.12605e-05,0.14078e-05,0.19167e-05,0.23729e-05,0.30161e-05]
  ka[:,12,11] = [0.11370e-05,0.91524e-06,0.11150e-05,0.14746e-05,0.20128e-05]
  ka[:,13,11] = [0.10511e-05,0.10014e-05,0.11405e-05,0.13852e-05,0.15576e-05]
  ka[:, 1,12] = [0.11184e-03,0.11117e-03,0.11327e-03,0.10989e-03,0.10910e-03]
  ka[:, 2,12] = [0.11379e-03,0.11322e-03,0.11555e-03,0.11462e-03,0.11135e-03]
  ka[:, 3,12] = [0.11508e-03,0.11459e-03,0.11421e-03,0.11671e-03,0.11339e-03]
  ka[:, 4,12] = [0.11596e-03,0.11563e-03,0.11538e-03,0.11511e-03,0.11770e-03]
  ka[:, 5,12] = [0.11597e-03,0.11581e-03,0.11569e-03,0.11553e-03,0.11890e-03]
  ka[:, 6,12] = [0.11443e-03,0.11445e-03,0.11443e-03,0.11443e-03,0.11438e-03]
  ka[:, 7,12] = [0.10852e-03,0.10888e-03,0.10912e-03,0.10934e-03,0.10942e-03]
  ka[:, 8,12] = [0.93194e-04,0.94766e-04,0.95355e-04,0.95090e-04,0.94926e-04]
  ka[:, 9,12] = [0.11836e-05,0.16115e-05,0.12883e-05,0.14202e-05,0.16541e-05]
  ka[:,10,12] = [0.18748e-05,0.34401e-05,0.39984e-05,0.44576e-05,0.33683e-05]
  ka[:,11,12] = [0.29890e-06,0.48741e-06,0.66276e-06,0.99698e-06,0.19230e-05]
  ka[:,12,12] = [0.15034e-06,0.39966e-06,0.56523e-06,0.70494e-06,0.10046e-05]
  ka[:,13,12] = [0.15016e-06,0.25751e-06,0.48928e-06,0.63534e-06,0.93575e-06]
  ka[:, 1,13] = [0.17305e-03,0.17234e-03,0.17389e-03,0.17055e-03,0.16974e-03]
  ka[:, 2,13] = [0.18170e-03,0.18075e-03,0.18265e-03,0.18138e-03,0.17772e-03]
  ka[:, 3,13] = [0.18990e-03,0.18892e-03,0.18776e-03,0.18950e-03,0.18494e-03]
  ka[:, 4,13] = [0.19649e-03,0.19552e-03,0.19424e-03,0.19281e-03,0.19464e-03]
  ka[:, 5,13] = [0.20197e-03,0.20109e-03,0.19993e-03,0.19856e-03,0.20092e-03]
  ka[:, 6,13] = [0.20595e-03,0.20549e-03,0.20452e-03,0.20331e-03,0.20199e-03]
  ka[:, 7,13] = [0.20703e-03,0.20710e-03,0.20649e-03,0.20552e-03,0.20428e-03]
  ka[:, 8,13] = [0.19874e-03,0.19767e-03,0.19696e-03,0.19655e-03,0.19591e-03]
  ka[:, 9,13] = [0.20434e-04,0.23398e-04,0.27400e-04,0.32409e-04,0.38451e-04]
  ka[:,10,13] = [0.18617e-05,0.99513e-06,0.10554e-05,0.16516e-05,0.37792e-05]
  ka[:,11,13] = [0.12517e-06,0.29518e-06,0.77058e-06,0.11660e-05,0.15349e-05]
  ka[:,12,13] = [0.12734e-06,0.36524e-06,0.66699e-06,0.10362e-05,0.14158e-05]
  ka[:,13,13] = [0.12431e-06,0.39389e-06,0.67331e-06,0.10292e-05,0.14448e-05]
  ka[:, 1,14] = [0.29365e-03,0.29046e-03,0.29008e-03,0.28509e-03,0.28286e-03]
  ka[:, 2,14] = [0.31990e-03,0.31668e-03,0.31617e-03,0.31332e-03,0.30885e-03]
  ka[:, 3,14] = [0.34787e-03,0.34432e-03,0.34112e-03,0.34052e-03,0.33589e-03]
  ka[:, 4,14] = [0.37401e-03,0.37027e-03,0.36696e-03,0.36394e-03,0.36355e-03]
  ka[:, 5,14] = [0.39840e-03,0.39446e-03,0.39082e-03,0.38763e-03,0.38760e-03]
  ka[:, 6,14] = [0.42165e-03,0.41729e-03,0.41335e-03,0.41006e-03,0.40721e-03]
  ka[:, 7,14] = [0.44257e-03,0.43782e-03,0.43364e-03,0.43014e-03,0.42736e-03]
  ka[:, 8,14] = [0.45299e-03,0.44953e-03,0.44586e-03,0.44260e-03,0.44006e-03]
  ka[:, 9,14] = [0.40190e-03,0.39751e-03,0.39238e-03,0.38812e-03,0.38612e-03]
  ka[:,10,14] = [0.64278e-05,0.18248e-05,0.16996e-05,0.31086e-06,0.16836e-06]
  ka[:,11,14] = [0.14350e-05,0.94778e-06,0.41349e-06,0.20817e-06,0.20238e-06]
  ka[:,12,14] = [0.16805e-05,0.15323e-05,0.62348e-06,0.99743e-07,0.12977e-06]
  ka[:,13,14] = [0.16858e-05,0.17103e-05,0.80574e-06,0.15825e-06,0.15032e-06]
  ka[:, 1,15] = [0.52181e-03,0.51578e-03,0.51251e-03,0.50356e-03,0.49731e-03]
  ka[:, 2,15] = [0.59491e-03,0.58822e-03,0.58413e-03,0.57646e-03,0.56692e-03]
  ka[:, 3,15] = [0.67653e-03,0.66881e-03,0.66126e-03,0.65540e-03,0.64461e-03]
  ka[:, 4,15] = [0.76388e-03,0.75456e-03,0.74556e-03,0.73649e-03,0.72840e-03]
  ka[:, 5,15] = [0.85507e-03,0.84417e-03,0.83378e-03,0.82338e-03,0.81349e-03]
  ka[:, 6,15] = [0.95034e-03,0.93798e-03,0.92553e-03,0.91287e-03,0.89957e-03]
  ka[:, 7,15] = [0.10496e-02,0.10352e-02,0.10206e-02,0.10054e-02,0.98958e-03]
  ka[:, 8,15] = [0.11507e-02,0.11337e-02,0.11169e-02,0.10991e-02,0.10806e-02]
  ka[:, 9,15] = [0.12408e-02,0.12207e-02,0.11996e-02,0.11773e-02,0.11531e-02]
  ka[:,10,15] = [0.12042e-03,0.11501e-03,0.11424e-03,0.11450e-03,0.13219e-03]
  ka[:,11,15] = [0.68914e-06,0.83960e-06,0.74591e-06,0.18660e-05,0.32503e-05]
  ka[:,12,15] = [0.35963e-07,0.46256e-06,0.56223e-06,0.98816e-06,0.92366e-06]
  ka[:,13,15] = [0.36605e-07,0.56591e-06,0.84008e-06,0.86042e-06,0.68452e-06]
  ka[:, 1,16] = [0.76517e-03,0.75944e-03,0.76010e-03,0.76100e-03,0.76498e-03]
  ka[:, 2,16] = [0.92375e-03,0.91357e-03,0.90997e-03,0.90997e-03,0.90993e-03]
  ka[:, 3,16] = [0.11142e-02,0.10974e-02,0.10835e-02,0.10789e-02,0.10748e-02]
  ka[:, 4,16] = [0.13278e-02,0.13025e-02,0.12802e-02,0.12631e-02,0.12539e-02]
  ka[:, 5,16] = [0.15712e-02,0.15343e-02,0.15017e-02,0.14713e-02,0.14497e-02]
  ka[:, 6,16] = [0.18525e-02,0.17982e-02,0.17525e-02,0.17101e-02,0.16714e-02]
  ka[:, 7,16] = [0.21731e-02,0.20986e-02,0.20340e-02,0.19757e-02,0.19210e-02]
  ka[:, 8,16] = [0.25325e-02,0.24346e-02,0.23473e-02,0.22687e-02,0.21950e-02]
  ka[:, 9,16] = [0.29269e-02,0.28006e-02,0.26863e-02,0.25805e-02,0.24878e-02]
  ka[:,10,16] = [0.29442e-02,0.27008e-02,0.23913e-02,0.21437e-02,0.18865e-02]
  ka[:,11,16] = [0.23220e-05,0.22310e-04,0.48349e-04,0.67183e-04,0.88908e-04]
  ka[:,12,16] = [0.22857e-05,0.11848e-04,0.42066e-04,0.67613e-04,0.86033e-04]
  ka[:,13,16] = [0.22823e-05,0.69105e-05,0.36212e-04,0.66247e-04,0.85488e-04]
end


module QCCDevControl_Types
export GateZone, Junction, AuxZone, Qubit, LoadingZone, QubitStatus, typesSizes, JunctionType
export Time_t, QCCDevControl

using LightGraphs
using ..QCCDevDes_Types

#Types of zones
const zoneTypes = Set([:junction,:loadingZone, :auxZone, :gateZone])

# Possible Qubits Status (not used)
const QubitStatus = Set([:inLoadingZone, :inGateZone])

# Supported junction types with corresponding sizes
const JunctionType = Set([:T, :Y, :X ])
const typesSizes = Dict(:T => 3, :Y => 3, :X => 4)

"""
Type for time inside the qdev, in [change if necessary]   10^{-10}
seconds, i.e., ns/10.  All times are ≥0; negative value of expressions
of this type are errors (and may carry local error information).
"""
const Time_t = Int64

"""
Struct for junction.
id: Junction ID.
type: Type of the junction. Each type may define how the junction works differently
ends: Array of its connections Id's
Throws ArgumentError if junction type doesn't match with number of ends.
"""
struct Junction
    id::Symbol
    type::Symbol
    zoneType::Symbol
    ends::Array{Symbol}
    function Junction(id::Symbol, type::Symbol, ends::Array{Symbol})
        type in JunctionType || throw(ArgumentError("Junction type $type not supported"))
        if length(ends) != typesSizes[type]
            throw(ArgumentError("Junction with ID $id of type $type has $(length(ends)) ends." *
            " It should have $(typesSizes[type]) ends."))
        end
        return new(id, type, :junction, ends)
    end
end

"""
Struct for the qubits.
id: qubit ID
status: current qubit status
    - moving
    - resting
    - waitingDecongestion
    - gateApplied
position: current qubit position
destination: qubit destination, it could not have any
"""
mutable struct Qubit
    id::Int
    status::Symbol
    position::Symbol
    destination::Union{Nothing,Symbol}
    Qubit(id::Int, position::Symbol) = new(id, :inLoadingZone, position, nothing)
end

"""
Struct for the auxiliary zones.
id: trap identifier
capacity: maximum qubits in the trap
chain: Ordered Qubits in the trap (from end0 to end1)
end0 & end1: Trap endings
"""
mutable struct LoadingZone
    id::Symbol
    zoneType::Symbol
    end0::Union{Symbol, Nothing}
    end1::Union{Symbol, Nothing}
    hole::Union{Int, Nothing}
    LoadingZone(id, end0, end1) = end0 == end1 && (!isnothing(end0) || !isnothing(end1)) ?
    throw(ArgumentError("In loading zone $id : \"end0\" and \"end1\" must be different")) :
    new(id, :loadingZone, end0, end1, nothing)
end

"""
Struct for the auxiliary zones.
id: trap identifier
capacity: maximum qubits in the trap
chain: Ordered Qubits in the trap (from end0 to end1)
end0 & end1: Trap endings
"""
struct AuxZone
    id::Symbol
    zoneType::Symbol
    capacity::Int64
    end0::Union{Symbol, Nothing}
    end1::Union{Symbol, Nothing}
    chain::Array{Array{Int64,1},1}
    AuxZone(id, capacity, end0, end1) = end0 == end1 && (!isnothing(end0) || !isnothing(end1)) ?
    throw(ArgumentError("In aux zone $id : \"end0\" and \"end1\" must be different")) :
    new(id, :auxZone, capacity, end0, end1, [])
end

"""
Struct for the gate zones.
id: trap identifier
capacity: maximum qubits in the trap
chain: Ordered Qubits in the trap (from end0 to end1)
end0 & end1: Trap endings
"""
struct GateZone
    id::Symbol
    zoneType::Symbol
    capacity::Int64
    end0::Union{Symbol, Nothing}
    end1::Union{Symbol, Nothing}
    chain::Array{Array{Int64,1},1}
    GateZone(id, capacity, end0, end1) = end0 == end1 && (!isnothing(end0) || !isnothing(end1)) ?
    throw(ArgumentError("In gate zone $id : \"end0\" and \"end1\" must be different")) :
    new(id, :gateZone, capacity, end0, end1, [])
end

mutable struct QCCDevControl
    dev            ::QCCDevDescription

    simulate       ::Symbol                   # one of `:No`, `:PureStates`, `:MixedStates`
    qnoise_esimate ::Bool                     # whether estimation of noise takes place

    t_now          ::Time_t

    qubits      ::Dict{Int,Qubit}
    gateZones      ::Dict{Symbol,GateZone}
    junctions      ::Dict{Symbol,Junction}
    auxZones       ::Dict{Symbol,AuxZone}
    loadingZones   ::Dict{Symbol,LoadingZone}

    # graph          ::SimpleGraph{Int64}

    QCCDevControl(dev, simulate, qnoise_estimate, gateZones, junctions, auxZones, loadingZones) =
            new(dev, simulate, qnoise_estimate, 0, Dict{String,Qubit}(),
            gateZones, junctions, auxZones, loadingZones)

end

end


"""
Translate mouse or modifier state into the corresponding XCB value.
"""
function state_xcb end

state_xcb(s::MouseState) = .&(UInt[s.left, s.middle, s.right, s.scroll_up, s.scroll_down, s.any] .* UInt[XCB_BUTTON_MASK_1, XCB_BUTTON_MASK_2, XCB_BUTTON_MASK_3, XCB_BUTTON_MASK_4, XCB_BUTTON_MASK_5, XCB_BUTTON_MASK_ANY]...)
state_xcb(s::KeyModifierState) = sum(2 .^ (0:3) .* Int[s.shift, s.ctrl, s.alt, s.super])
state_xcb(::Any) = 0

state_xcb(e::MouseEvent) = state_xcb(e.state)
state_xcb(e::KeyEvent) = state_xcb(e.modifiers)

"""
Translate an action into its corresponding XCB value.
"""
function response_type_xcb end

response_type_xcb(::Type{ButtonPressed}) = XCB_BUTTON_PRESS
response_type_xcb(::Type{ButtonReleased}) = XCB_BUTTON_RELEASE
response_type_xcb(::Type{KeyPressed}) = XCB_KEY_PRESS
response_type_xcb(::Type{KeyReleased}) = XCB_KEY_RELEASE
response_type_xcb(::Type{PointerMoves}) = XCB_MOTION_NOTIFY
response_type_xcb(::Type{PointerEntersWindow}) = XCB_ENTER_NOTIFY
response_type_xcb(::Type{PointerLeavesWindow}) = XCB_LEAVE_NOTIFY
response_type_xcb(::Type{Expose}) = XCB_EXPOSE
response_type_xcb(::Type{Resize}) = XCB_CONFIGURE_NOTIFY

event_type_xcb(::Type{<:Union{ButtonPressed, ButtonReleased}}) = xcb_button_press_event_t
event_type_xcb(::Type{<:Union{KeyPressed, KeyReleased}}) = xcb_key_press_event_t
event_type_xcb(::Type{PointerMoves}) = xcb_motion_notify_event_t
event_type_xcb(::Type{<:Union{PointerEntersWindow, PointerLeavesWindow}}) = xcb_enter_notify_event_t
event_type_xcb(::Type{ResizeEvent}) = xcb_configure_notify_event_t
event_type_xcb(::Type{ExposeEvent}) = xcb_expose_event_t

button_xcb(::ButtonLeft) = XCB_BUTTON_INDEX_1
button_xcb(::ButtonMiddle) = XCB_BUTTON_INDEX_2
button_xcb(::ButtonRight) = XCB_BUTTON_INDEX_3
button_xcb(::ButtonScrollUp) = XCB_BUTTON_INDEX_4
button_xcb(::ButtonScrollDown) = XCB_BUTTON_INDEX_5

detail_xcb(wm::XWindowManager, e::MouseEvent) = button_xcb(e.button)
detail_xcb(wm::XWindowManager, e::PointerMovesEvent) = XCB_MOTION_NORMAL
detail_xcb(wm::XWindowManager, e::PointerEntersWindowEvent) = XCB_ENTER_NOTIFY
detail_xcb(wm::XWindowManager, e::PointerLeavesWindowEvent) = XCB_LEAVE_NOTIFY
detail_xcb(wm::XWindowManager, e::EventData) = 0
detail_xcb(wm::XWindowManager, e::EventDetails) = detail_xcb(wm, e.data)
detail_xcb(wm::XWindowManager, e::EventDetails{<:KeyEvent}) = keycode(wm, e)

event_xcb(wm::XWindowManager, e::EventDetails) = event_type_xcb(action(e))(
    response_type_xcb(action(e)),
    detail_xcb(wm, e),
    0,
    e.time,
    e.win.parent_id,
    e.win.id,
    0,
    0,
    0,
    e.location...,
    state_xcb(e.data),
    true,
    false
)

event_xcb(wm::XWindowManager, e::EventDetails{<:ExposeEvent}) = event_type_xcb(action(e))(
    response_type_xcb(action(e)),
    0,
    0,
    e.win.id,
    e.location...,
    extent(e.win)...,
    0,
    0
)

event_xcb(wm::XWindowManager, e::EventDetails{<:ResizeEvent}) = event_type_xcb(action(e))(
    response_type_xcb(action(e)),
    0,
    0,
    e.win.id,
    e.win.id,
    0,
    e.location...,
    extent(e.win)...,
    0,
    0,
    0
)

send_event(wm::XWindowManager, e::EventDetails) = send_event(e.win, event_xcb(wm, e))

function send_event(win::XCBWindow, event)
    ref = Ref(event)
    GC.@preserve ref begin
        event_ptr = Base.reinterpret(Cstring, Base.unsafe_convert(Ptr{typeof(event)}, ref))
        @flush @check :error xcb_send_event(win.conn, false, win.id, 0, event_ptr)
    end
end

hex(x) = "0x$(string(x, base=16))"

function send(wm::XWindowManager, win::XCBWindow; location=(0, 0))
    (event; location=location) -> send_event(wm, EventDetails(event, location, floor(time()), win))
end


# Functions for exporting molecular dynamics simulation results to visualization tools

"""
    write_nbs_animation(result::NBSResult, filename::String)

Animate an `NBSResult` and store the result in a .gif file.
"""
function write_nbs_animation(result::NBSResult, filename::String)
    animate(result.result, filename)
end

"""
    write_ase_trajectory(result::MolecularDynamicsResult, element::DFTK.Element, lattice, filename::String)

Write the trajectory of a `MolecularDynamicsResult` to a .traj file.

The file can be visualized by running `ase gui <filename>` on the command line.
"""
# TODO: support multiple species using atom data
function write_ase_trajectory(result::MolecularDynamicsResult, element::DFTK.Element, lattice, filename::String)
    ase = pyimport_e("ase")
    if ispynull(ase)
        @warn "ASE is not installed, skipping write_trajectory"
    else
        filename = abspath(expanduser(filename))
        if isfile(filename)
            @info "overwriting trajectory at" filename
            rm(filename)
        else
            @info "writing trajectory to" filename
        end
        traj = pyimport("ase.io.trajectory").Trajectory(filename, "a")
        for t ∈ 1:length(get_time_range(result))
            traj.write(ase_atoms(lattice, dftk_atoms(get_system(result, t), element)))
        end
        traj.close()
    end
end


import CSV
import DataFrames
import JSON

# Kept for compatibility with legacy code.
function create_profile_data(number_of_periods, data, loadprofile = ones(length(data["load"]),number_of_periods), genprofile = ones(length(data["gen"]),number_of_periods))
    _FP.make_time_series(data, number_of_periods; loadprofile = permutedims(loadprofile), genprofile = permutedims(genprofile))
end

function create_profile_data_italy!(data)

    hours = _FP.dim_length(data, :hour)
    scenarios = _FP.dim_length(data, :scenario)

    genprofile = ones(length(data["gen"]), hours*scenarios)
    loadprofile = ones(length(data["load"]), hours*scenarios)

    monte_carlo = get(_FP.dim_meta(data, :scenario), "mc", false)

    for (s, scnr) in _FP.dim_prop(data, :scenario)
        pv_sicily, pv_south_central, wind_sicily = read_res_data(s; mc = monte_carlo)
        demand_center_north_pu, demand_north_pu, demand_center_south_pu, demand_south_pu, demand_sardinia_pu = read_demand_data(s; mc = monte_carlo)

        start_idx = (s-1) * hours
        if monte_carlo == false
            for h in 1 : hours
                h_idx = scnr["start"] + ((h-1) * 3600000)
                genprofile[3, start_idx + h] = pv_south_central["data"]["$h_idx"]["electricity"]
                genprofile[5, start_idx + h] = pv_sicily["data"]["$h_idx"]["electricity"]
                genprofile[6, start_idx + h] = wind_sicily["data"]["$h_idx"]["electricity"]
            end
        else
            genprofile[3, start_idx + 1 : start_idx + hours] = pv_south_central[1: hours]
            genprofile[5, start_idx + 1 : start_idx + hours] = pv_sicily[1: hours]
            genprofile[6, start_idx + 1 : start_idx + hours] = wind_sicily[1: hours]
        end
        loadprofile[:, start_idx + 1 : start_idx + hours] = [demand_center_north_pu'; demand_north_pu'; demand_center_south_pu'; demand_south_pu'; demand_sardinia_pu'][:, 1: hours]
        # loadprofile[:, start_idx + 1 : start_idx + number_of_hours] = repeat([demand_center_north_pu'; demand_north_pu'; demand_center_south_pu'; demand_south_pu'; demand_sardinia_pu'][:, 1],1,number_of_hours)
    end
    # Add bus locations to data dictionary
    data["bus"]["1"]["lat"] = 43.4894; data["bus"]["1"]["lon"] = 11.7946; # Italy central north
    data["bus"]["2"]["lat"] = 45.3411; data["bus"]["2"]["lon"] =  9.9489; # Italy north
    data["bus"]["3"]["lat"] = 41.8218; data["bus"]["3"]["lon"] = 13.8302; # Italy central south
    data["bus"]["4"]["lat"] = 40.5228; data["bus"]["4"]["lon"] = 16.2155; # Italy south
    data["bus"]["5"]["lat"] = 40.1717; data["bus"]["5"]["lon"] =  9.0738; # Sardinia
    data["bus"]["6"]["lat"] = 37.4844; data["bus"]["6"]["lon"] = 14.1568; # Sicily
    # Return info
    return data, loadprofile, genprofile
end

function create_profile_data_germany!(data)

    hours = _FP.dim_length(data, :hour)
    scenarios = _FP.dim_length(data, :scenario)

    genprofile = ones(length(data["gen"]), hours*scenarios)
    loadprofile = ones(length(data["load"]), hours*scenarios)

    monte_carlo = get(_FP.dim_meta(data, :scenario), "mc", false)

    for (s, scnr) in _FP.dim_prop(data, :scenario)
        wind_profile = read_res_data(s; mc = monte_carlo, country = "de")
        demand_profile = read_demand_data(s; mc = monte_carlo, country = "de")
        start_idx = (s-1) * hours
        if monte_carlo == false
            for h in 1 : hours
                h_idx = scnr["start"] + ((h-1) * 3600000)
                genprofile[2, start_idx + h] = wind_profile["2"]["data"]["$h_idx"]["electricity"]
                genprofile[4, start_idx + h] = wind_profile["5"]["data"]["$h_idx"]["electricity"]
                genprofile[20, start_idx + h] = wind_profile["67"]["data"]["$h_idx"]["electricity"]
                if length(data["gen"]) > 20
                    genprofile[21, start_idx + h] = wind_profile["23"]["data"]["$h_idx"]["electricity"]
                elseif length(data["gen"]) > 21
                    genprofile[22, start_idx + h] = wind_profile["54"]["data"]["$h_idx"]["electricity"]
                end
            end
        end
        loadprofile[:, start_idx + 1 : start_idx + hours] .= repeat(demand_profile[1:hours]', size(loadprofile, 1))
    end
    # Return info
    return data, loadprofile, genprofile
end

"Create load and generation profiles for CIGRE distribution network."
function create_profile_data_cigre(data, number_of_hours; start_period = 1, scale_load = 1.0, scale_gen = 1.0, file_profiles_pu = normpath(@__DIR__,"..","data","cigre_mv_eu","time_series","CIGRE_profiles_per_unit.csv"))

    ## Fixed parameters

    file_load_ind = normpath(@__DIR__,"..","data","cigre_mv_eu","CIGRE_industrial_loads.csv")
    file_load_res = normpath(@__DIR__,"..","data","cigre_mv_eu","CIGRE_residential_loads.csv")
    scale_unit    = 0.001 # scale factor from CSV power data to FlexPlan power base: here converts from kVA to MVA

    ## Import data

    load_ind    = CSV.read(file_load_ind, DataFrames.DataFrame)
    load_res    = CSV.read(file_load_res, DataFrames.DataFrame)
    profiles_pu = CSV.read(
        file_profiles_pu,
        DataFrames.DataFrame;
        skipto = start_period + 1, # +1 is for header line
        limit = number_of_hours,
        ntasks = 1 # To ensure exact row limit is applied
    )
    if DataFrames.nrow(profiles_pu) < number_of_hours
        Memento.error(_LOGGER, "insufficient number of rows in file \"$file_profiles_pu\" ($number_of_hours requested, $(DataFrames.nrow(profiles_pu)) found)")
    end
    DataFrames.select!(profiles_pu,
        :industrial_load  => :load_ind,
        :residential_load => :load_res,
        :photovoltaic     => :pv,
        :wind_turbine     => :wind,
        :fuel_cell        => :fuel_cell,
        :CHP_diesel       => :chp_diesel,
        :CHP_fuel_cell    => :chp_fuel_cell
    )
    profiles_pu = Dict(pairs(eachcol(profiles_pu)))

    ## Prepare output structure

    extradata = Dict{String,Any}()

    ## Loads

    # Compute active and reactive power base of industrial loads
    DataFrames.rename!(load_ind, [:bus, :s, :cosϕ])
    load_ind.p_ind = scale_load * scale_unit * load_ind.s .* load_ind.cosϕ
    load_ind.q_ind = scale_load * scale_unit * load_ind.s .* sin.(acos.(load_ind.cosϕ))
    DataFrames.select!(load_ind, :bus, :p_ind, :q_ind)

    # Compute active and reactive power base of residential loads
    DataFrames.rename!(load_res, [:bus, :s, :cosϕ])
    load_res.p_res = scale_load * scale_unit * load_res.s .* load_res.cosϕ
    load_res.q_res = scale_load * scale_unit * load_res.s .* sin.(acos.(load_res.cosϕ))
    DataFrames.select!(load_res, :bus, :p_res, :q_res)

    # Create a table of industrial and residential power bases, indexed by the load ids used by `data`
    load_base = coalesce.(DataFrames.outerjoin(load_ind, load_res; on=:bus), 0.0)
    load_base.bus = string.(load_base.bus)
    bus_load_lookup = Dict{String,String}()
    for (l, load) in data["load"]
        bus_load_lookup["$(load["load_bus"])"] = l
    end
    DataFrames.transform!(load_base, :bus => DataFrames.ByRow(b -> bus_load_lookup[b]) => :load_id)

    # Compute active and reactive power profiles of each load
    extradata["load"] = Dict{String,Any}()
    for l in eachrow(load_base)
        extradata["load"][l.load_id] = Dict{String,Any}()
        extradata["load"][l.load_id]["pd"] = l.p_ind .* profiles_pu[:load_ind] .+ l.p_res .* profiles_pu[:load_res]
        extradata["load"][l.load_id]["qd"] = l.q_ind .* profiles_pu[:load_ind] .+ l.q_res .* profiles_pu[:load_res]
    end

    ## Generators

    # Define a Dict for the technology of generators, indexed by the gen ids used by `data`
    gen_tech = Dict{String,Symbol}()
    gen_tech["1"]  = :pv
    gen_tech["2"]  = :pv
    gen_tech["3"]  = :pv
    gen_tech["4"]  = :fuel_cell
    gen_tech["5"]  = :pv
    gen_tech["6"]  = :wind
    gen_tech["7"]  = :pv
    gen_tech["8"]  = :pv
    gen_tech["9"]  = :chp_diesel
    gen_tech["10"] = :chp_fuel_cell
    gen_tech["11"] = :pv
    gen_tech["12"] = :fuel_cell
    gen_tech["13"] = :pv

    # Compute active and reactive power profiles of each generator
    extradata["gen"]  = Dict{String,Any}()
    for (g, gen) in data["gen"]
        if haskey(gen_tech, g)
            extradata["gen"][g] = Dict{String,Any}()
            extradata["gen"][g]["pmax"] = scale_gen * gen["pmax"] .* profiles_pu[gen_tech[g]]
            extradata["gen"][g]["pmin"] = scale_gen * gen["pmin"] .* profiles_pu[gen_tech[g]]
            extradata["gen"][g]["qmax"] = scale_gen * gen["qmax"] .* ones(number_of_hours)
            extradata["gen"][g]["qmin"] = scale_gen * gen["qmin"] .* ones(number_of_hours)
        end
    end

    return extradata
end

function read_demand_data(year; mc = false, country = "it")
    if country == "it"
        if mc == false
            if year > 2
                error("Only 2 scenarios are supported")
            end
            y = year + 2017
            # Read demand CSV files
            demand_north = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/demand_north_$y.csv"),DataFrames.DataFrame)[:,3]
            demand_center_north = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/demand_center_north_$y.csv"),DataFrames.DataFrame)[:,3]
            demand_center_south = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/demand_center_south_$y.csv"),DataFrames.DataFrame)[:,3]
            demand_south = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/demand_south_$y.csv"),DataFrames.DataFrame)[:,3]
            demand_sardinia = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/demand_sardinia_$y.csv"),DataFrames.DataFrame)[:,3]

            # Convert demand_profile to pu of maxximum
            demand_north_pu = demand_north ./ maximum(demand_north)
            demand_center_north_pu = demand_center_north ./ maximum(demand_center_north)
            demand_south_pu = demand_south ./ maximum(demand_south)
            demand_center_south_pu = demand_center_south ./ maximum(demand_center_south)
            demand_sardinia_pu = demand_sardinia ./ maximum(demand_sardinia)
        else
            if year > 35
                error("Only 35 scenarios are supported")
            end
            y = year - 1
            demand_north_pu = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_demand_$y.csv"),DataFrames.DataFrame)[:,3]
            demand_center_north_pu = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_demand_$y.csv"),DataFrames.DataFrame)[:,2]
            demand_center_south_pu = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_demand_$y.csv"),DataFrames.DataFrame)[:,4]
            demand_south_pu = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_demand_$y.csv"),DataFrames.DataFrame)[:,5]
            demand_sardinia_pu = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_demand_$y.csv"),DataFrames.DataFrame)[:,6]
        end

        return demand_north_pu, demand_center_north_pu, demand_center_south_pu, demand_south_pu, demand_sardinia_pu
    elseif country == "de"
        if year > 3
            error("Only 3 scenarios are supported")
        end
        y = year + 2016
        demand = CSV.read(normpath(@__DIR__,"../../test/data/case67/time_series/demand$y.csv"),DataFrames.DataFrame)[:,3]
        demand_pu = demand ./ maximum(demand)
        return demand_pu[1:4:end]
    end
end

function read_res_data(year; mc = false, country = "it")
    if country == "it"
        if mc == false
            if year > 2
                error("Only 2 scenarios are supported")
            end
            y = year + 2017
            pv_sicily = Dict()
            open(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/pv_sicily_$y.json")) do f
                dicttxt = read(f, String)  # file information to string
                pv_sicily = JSON.parse(dicttxt)  # parse and transform data
            end

            pv_south_central = Dict()
            open(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/pv_south_central_$y.json")) do f
                dicttxt = read(f, String)  # file information to string
                pv_south_central = JSON.parse(dicttxt)  # parse and transform data
            end

            wind_sicily = Dict()
            open(normpath(@__DIR__,"../../test/data/case6/time_series/mc_false/wind_sicily_$y.json")) do f
                dicttxt = read(f, String)  # file information to string
                wind_sicily = JSON.parse(dicttxt)  # parse and transform data
            end
        else
            if year > 35
                error("Only 35 scenarios are supported")
            end
            y = year - 1
            pv_sicily = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_PV_$y.csv"),DataFrames.DataFrame)[:,7]
            pv_south_central = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_PV_$y.csv"),DataFrames.DataFrame)[:,4]
            wind_sicily = CSV.read(normpath(@__DIR__,"../../test/data/case6/time_series/mc_true/case_6_wind_$y.csv"),DataFrames.DataFrame)[:,7]
        end
        return pv_sicily, pv_south_central, wind_sicily
    elseif country == "de"
        if year > 3
            error("Only 3 scenarios are supported")
        end
        y = year + 2016
        wind_profile = Dict{String, Any}()
        open(normpath(@__DIR__,"../../test/data/case67/time_series/wind_bus2_$y.json")) do f
            dicttxt = read(f, String)  # file information to string
            wind_profile["2"] = JSON.parse(dicttxt)  # parse and transform data
        end
        open(normpath(@__DIR__,"../../test/data/case67/time_series/wind_bus5_$y.json")) do f
            dicttxt = read(f, String)  # file information to string
            wind_profile["5"]  = JSON.parse(dicttxt)  # parse and transform data
        end
        open(normpath(@__DIR__,"../../test/data/case67/time_series/wind_bus23_$y.json")) do f
            dicttxt = read(f, String)  # file information to string
            wind_profile["23"]  = JSON.parse(dicttxt)  # parse and transform data
        end
        open(normpath(@__DIR__,"../../test/data/case67/time_series/wind_bus54_$y.json")) do f
            dicttxt = read(f, String)  # file information to string
            wind_profile["54"]  = JSON.parse(dicttxt)  # parse and transform data
        end
        open(normpath(@__DIR__,"../../test/data/case67/time_series/wind_bus67_$y.json")) do f
            dicttxt = read(f, String)  # file information to string
            wind_profile["67"]  = JSON.parse(dicttxt)  # parse and transform data
        end
        return wind_profile
    end
end


import Knet.Ops20: conv4, conv4x, conv4w, pool, poolx
using CUDA: CUDA, CUDNN, Mem
using Knet.KnetArrays: DevArray, Cptr

function conv4(w::R,x::R; handle=CUDNN.handle(), alpha=1,
               o...) where {T,R<:DevArray{T}} # padding=0, stride=1, dilation=1, mode=0, group=1
    beta=0 # nonzero beta does not make sense when we create y
    y = similar(x, cdims(w,x;o...))
    (algo,workSpace) = conv4_algo(w, x, y; handle=handle, o...)
    CUDNN.cudnnConvolutionForward(handle,Ref(T(alpha)),TD(x),x,FD(w),w,CD(w,x;o...),algo,workSpace,bytes(workSpace),Ref(T(beta)),TD(y),y)
    return y
end

function conv4x(w::R,x::R,dy::R; handle=CUDNN.handle(), alpha=1,
                   o...) where {T,R<:DevArray{T}} # padding=0, stride=1, dilation=1, mode=0, group=1
    beta = 0
    dx = similar(x)
    (algo,workSpace) = conv4x_algo(w,x,dy,dx; handle=handle, o...)
    CUDNN.cudnnConvolutionBackwardData(handle,Ref(T(alpha)),FD(w),w,TD(dy),dy,CD(w,x;o...),algo,workSpace,bytes(workSpace),Ref(T(beta)),TD(dx),dx)
    return dx
end

function conv4w(w::R,x::R,dy::R; handle=CUDNN.handle(), alpha=1,
                   o...) where {T,R<:DevArray{T}} # padding=0, stride=1, dilation=1, mode=0, group=1
    beta = 0
    dw = similar(w)
    (algo,workSpace) = conv4w_algo(w,x,dy,dw;handle=handle,o...)
    CUDNN.cudnnConvolutionBackwardFilter(handle,Ref(T(alpha)),TD(x),x,TD(dy),dy,CD(w,x;o...),algo,workSpace,bytes(workSpace),Ref(T(beta)),FD(dw),dw)
    return dw
end

function pool(x::R; handle=CUDNN.handle(), alpha=1,
                 o...) where {T,R<:DevArray{T}} # window=2, padding=0, stride=window, mode=0, maxpoolingNanOpt=0
    y = similar(x, pdims(x; o...))
    beta = 0
    CUDNN.cudnnPoolingForward(handle,PD(x;o...),Ref(T(alpha)),TD(x),x,    Ref(T(beta)),TD(y),y)
    return y
end

function poolx(x::R,y::R,dy::R; handle=CUDNN.handle(), alpha=1, mode=0,
                  o...) where {T,R<:DevArray{T}} # window=2, padding=0, stride=window, maxpoolingNanOpt=0
    dx = similar(x)
    beta = 0
    CUDNN.cudnnPoolingBackward(handle,PD(x;mode=mode,o...),Ref(T(alpha)),TD(y),y,TD(dy),dy,TD(x),x,Ref(T(beta)),TD(dx),dx)
    return dx
end

# cudnn descriptors: need to use mutable structs in order to have finalizers.

mutable struct TD; ptr; end
TD(a::DevArray{T}) where {T} = TD(T,size(a))
TD(T::Type, dims::Integer...) = TD(T, dims)
function TD(T::Type, dims)
    d = Cptr[0]
    CUDNN.cudnnCreateTensorDescriptor(d)
    n = length(dims)
    sz = [Cint(dims[i]) for i=n:-1:1]
    st = similar(sz); st[n] = 1
    for i=(n-1):-1:1; st[i] = st[i+1] * sz[i+1]; end
    dt = CUDNN.cudnnDataType_t(DT(T))
    CUDNN.cudnnSetTensorNdDescriptor(d[1], dt, n, sz, st)
    td = TD(d[1])
    finalizer(x->CUDNN.cudnnDestroyTensorDescriptor(x.ptr), td)
    return td
end

mutable struct FD; ptr; end
FD(a::DevArray{T}) where {T}=FD(T,size(a))
FD(T::Type, dims::Integer...) = FD(T,dims)
function FD(T::Type, dims)
    d = Cptr[0]
    CUDNN.cudnnCreateFilterDescriptor(d)
    n = length(dims)
    sz = [Cint(dims[i]) for i=n:-1:1]
    dt = CUDNN.cudnnDataType_t(DT(T))
    tf = CUDNN.cudnnTensorFormat_t(0)
    CUDNN.cudnnSetFilterNdDescriptor(d[1], dt, tf, n, sz)
    fd = FD(d[1])
    finalizer(x->CUDNN.cudnnDestroyFilterDescriptor(x.ptr), fd)
    return fd
end

mutable struct CD; ptr; end
function CD(w::DevArray,x::DevArray; padding=0, stride=1, dilation=1, mode=0, group=1)
    d = Cptr[0]
    CUDNN.cudnnCreateConvolutionDescriptor(d)
    cd = CD(d[1])
    nd = ndims(x)-2
    dt = CUDNN.cudnnDataType_t(DT(x))
    mode = CUDNN.cudnnConvolutionMode_t(mode)
    CUDNN.cudnnSetConvolutionNdDescriptor(cd,nd,cdsize(padding,nd),cdsize(stride,nd),cdsize(dilation,nd),mode,dt)
    CUDNN.cudnnSetConvolutionGroupCount(cd,group)
    finalizer(x->CUDNN.cudnnDestroyConvolutionDescriptor(x.ptr),cd)
    return cd
end

mutable struct PD; ptr; end
function PD(x::DevArray; window=2, padding=0, stride=window, mode=0, maxpoolingNanOpt=0)
    d = Cptr[0]
    CUDNN.cudnnCreatePoolingDescriptor(d)
    nd = ndims(x)-2
    mode = CUDNN.cudnnPoolingMode_t(mode)
    maxpoolingNanOpt = CUDNN.cudnnNanPropagation_t(maxpoolingNanOpt)
    CUDNN.cudnnSetPoolingNdDescriptor(d[1],mode,maxpoolingNanOpt,nd,cdsize(window,nd),cdsize(padding,nd),cdsize(stride,nd))
    pd = PD(d[1])
    finalizer(x->CUDNN.cudnnDestroyPoolingDescriptor(x.ptr), pd)
    return pd
end

import Base: unsafe_convert
unsafe_convert(::Type{Cptr}, td::TD)=td.ptr
unsafe_convert(::Type{Cptr}, fd::FD)=fd.ptr
unsafe_convert(::Type{Cptr}, cd::CD)=cd.ptr
unsafe_convert(::Type{Cptr}, pd::PD)=pd.ptr

# fill and reverse Cint array with padding etc. for cudnn calls
function cdsize(w, nd)
    if isa(w,Number)
        fill(Cint(w),nd)
    elseif length(w)==nd
        [ Cint(w[nd-i+1]) for i=1:nd ]
    else
        throw(DimensionMismatch("$w $nd"))
    end
end

DT(::DevArray{Float32})=Cint(0)
DT(::DevArray{Float64})=Cint(1)
DT(::DevArray{Float16})=Cint(2)
DT(::Type{Float32}) = Cint(0)
DT(::Type{Float64}) = Cint(1)
DT(::Type{Float16}) = Cint(2)

## Utilities to find a fast algorithm

struct cudnnConvolutionFwdAlgoPerf_t
    algo::Cint
    status::Cint
    time::Cfloat
    memory::Csize_t
    determinism::Cint
    mathType::Cint
    r1::Cint; r2::Cint; r3::Cint
end

const CUDNN_MAX_FIND = 100      # How many times can we call FindAlgorithm
const requestedAlgoCount = 10
const returnedAlgoCount = Cint[0]
bytes(x::DevArray{T}) where {T}=length(x)*sizeof(T)
gpufree() = Mem.info()[1] + (isdefined(CUDA,:pool) ? CUDA.pool[].cached_memory() : CUDA.cached_memory())

# This seems to cover a reasonable subset of the available algorithms
# The user can set this to 0 for a more memory-tight execution
maxWorkspaceSize(w,x,y) = min(gpufree() ÷ 10, bytes(x) * 100)

const conv4_algos = Dict()
function conv4_algo(w::R, x::R, y::R; handle=CUDNN.handle(), o...) where {T,R<:DevArray{T}}
    key = (T,size(w),size(x),o...)
    if haskey(conv4_algos, key)
        p = conv4_algos[key]
    elseif length(conv4_algos) >= CUDNN_MAX_FIND
        p = nothing
    else
        workSpace = similar(w, maxWorkspaceSize(w,x,y) ÷ sizeof(T))
        perfResults = Array{CUDNN.cudnnConvolutionFwdAlgoPerf_t}(undef,requestedAlgoCount)
        wd, xd, yd, cd = FD(w), TD(x), TD(y), CD(w,x;o...)
        CUDNN.cudnnFindConvolutionForwardAlgorithmEx(handle,xd,x,wd,w,cd,yd,y,requestedAlgoCount,returnedAlgoCount,perfResults,workSpace,bytes(workSpace))
        p = perfChoose(perfResults, returnedAlgoCount[1])
        conv4_algos[key] = p
        if p === nothing
            @warn "No good algo found for conv4$o: using default algo=0." maxlog=1
        end
    end
    if p === nothing
        return (CUDNN.cudnnConvolutionFwdAlgo_t(0), cudnnWorkSpace(w))
    else
        return (p.algo, cudnnWorkSpace(w,p.memory))
    end
end

const conv4w_algos = Dict()
function conv4w_algo(w::R,x::R,dy::R,dw::R; handle=CUDNN.handle(), o...) where {T,R<:DevArray{T}}
    key = (T,size(w),size(x),o...)
    if haskey(conv4w_algos, key)
        p = conv4w_algos[key]
    elseif length(conv4w_algos) >= CUDNN_MAX_FIND
        p = nothing
    else
        workSpace = similar(w, maxWorkspaceSize(w,x,dy) ÷ sizeof(T))
        perfResults = Array{CUDNN.cudnnConvolutionBwdFilterAlgoPerf_t}(undef,requestedAlgoCount)
        wd, xd, yd, cd = FD(dw), TD(x), TD(dy), CD(w,x;o...)
        CUDNN.cudnnFindConvolutionBackwardFilterAlgorithmEx(handle,xd,x,yd,dy,cd,wd,dw,requestedAlgoCount,returnedAlgoCount,perfResults,workSpace,bytes(workSpace))
        p = perfChoose(perfResults, returnedAlgoCount[1])
        conv4w_algos[key] = p
        if p === nothing
            @warn "No good algo found for conv4w$o: using default algo=0." maxlog=1
        end
    end
    if p === nothing
        return (CUDNN.cudnnConvolutionBwdFilterAlgo_t(0), cudnnWorkSpace(w))
    else
        return (p.algo, cudnnWorkSpace(w, p.memory))
    end
end

const conv4x_algos = Dict()
function conv4x_algo(w::R,x::R,dy::R,dx::R; handle=CUDNN.handle(), o...) where {T,R<:DevArray{T}}
    key = (T,size(w),size(x),o...)
    if haskey(conv4x_algos, key)
        p = conv4x_algos[key]
    elseif length(conv4x_algos) >= CUDNN_MAX_FIND
        p = nothing
    else
        workSpace = similar(w, maxWorkspaceSize(w,x,dy) ÷ sizeof(T))
        perfResults = Array{CUDNN.cudnnConvolutionBwdDataAlgoPerf_t}(undef,requestedAlgoCount)
        wd, xd, yd, cd = FD(w), TD(dx), TD(dy), CD(w,x;o...)
        CUDNN.cudnnFindConvolutionBackwardDataAlgorithmEx(handle,wd,w,yd,dy,cd,xd,dx,requestedAlgoCount,returnedAlgoCount,perfResults,workSpace,bytes(workSpace))
        p = perfChoose(perfResults, returnedAlgoCount[1])
        conv4x_algos[key] = p
        if p === nothing
            @warn "No good algo found for conv4x$o: using default algo=0." maxlog=1
        end
    end
    if p === nothing
        return (CUDNN.cudnnConvolutionBwdDataAlgo_t(0), cudnnWorkSpace(w))
    else
        return (p.algo, cudnnWorkSpace(w, p.memory))
    end
end

function perfChoose(ps, n)
    if n==length(ps)
        @warn "returnedAlgoCount==requestedAlgoCount" maxlog=1
    end
    (ibest,mbest,tbest) = (0,Inf,Inf)
    for i = 1:n
        # These metrics are written in a sorted fashion where the first element has the lowest compute time.
        if ps[i].status == 0 && ps[i].memory < mbest && ps[i].time < tbest * 1.1
            (ibest,mbest,tbest) = (i,ps[i].memory,ps[i].time)
        end
    end
    if ibest > 0
        return ps[ibest]
    else
        return nothing
    end
end

# Fresh workspace for every op is safer:
cudnnWorkSpace(w)=similar(w, 0)
cudnnWorkSpace(w,len)=similar(w, 1 + (len-1) ÷ sizeof(eltype(w)))

## Dimension helpers:

# outputDim = 1 + ( inputDim + 2*pad - (((filterDim-1)*dilation)+1) )/convolutionStride;

function cdims(w,x; padding=0, stride=1, dilation=1, o...)
    N = ndims(x)
    ntuple(N) do i
        if i < N-1
            pi = (if isa(padding,Number); padding; else padding[i]; end)
            si = (if isa(stride,Number); stride; else stride[i]; end)
            di = (if isa(dilation,Number); dilation; else dilation[i]; end)
            1 + (size(x,i) + 2*pi - (((size(w,i)-1)*di)+1)) ÷ si
        elseif i == N-1
            size(w,N)
        else # i == N
            size(x,N)
        end
    end
end

function pdims(x; window=2, padding=0, stride=window, o...)
    N = ndims(x)
    ntuple(N) do i
        if i < N-1
            wi = (if isa(window,Number); window; else window[i]; end)
            pi = (if isa(padding,Number); padding; else padding[i]; end)
            si = (if isa(stride,Number); stride; else stride[i]; end)
            1 + div(size(x,i) + 2*pi - wi, si)
        else
            size(x,i)
        end
    end
end