<commit_before>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#include "miopen/solver.hpp"
#include "miopen/handle.hpp"

namespace miopen {
namespace solver {

bool ConvOclDirectFwd3x3::IsApplicable(const ConvolutionContext& params) const
{
    if(!params.Is2d())
        return false;
    if(!(params.IsFp32() || params.IsFp16() || params.IsBfp16()))
        return false;

    return params.kernel_size_w == 3 && params.kernel_size_h == 3 && params.pad_w == 1 &&
           params.pad_h == 1 && params.kernel_stride_w == 1 && params.kernel_stride_h == 1 &&
           params.kernel_dilation_w == 1 && params.kernel_dilation_h == 1 &&
           params.group_counts == 1 && params.direction.IsForward() &&
           (params.out_width == 512 || params.out_width == 64 || params.out_width == 128 ||
            params.out_width == 256);
}

ConvSolution ConvOclDirectFwd3x3::GetSolution(const ConvolutionContext& params) const
{
    ConvSolution result;
    // size_t localMemSize = params.stream.GetLocalMemorySize();
    auto hw_wave_sz = 64;
    // auto dev_local_mem_sz = localMemSize; // in bytes
    int n_waves = 4;

    int wei_cstride = params.kernel_size_w * params.kernel_size_h;
    int wei_bstride = params.n_inputs * wei_cstride;

    result.out_pix_tile0   = 4;
    result.out_pix_tile1   = 2;
    result.n_stacks        = 1;
    result.n_out_pix_tiles = 4;
    int read_unit          = result.out_pix_tile0;
    //	std::string READ_TYPE = (read_unit == 1) ? "_FLOAT" : "_FLOAT" +
    // std::to_string(static_cast<long long>(read_unit));
    // MD: read_unit is never == 1
    std::string READ_TYPE = "_FLOAT" + std::to_string(static_cast<long long>(read_unit));

    int GRP_SZ = hw_wave_sz * n_waves;

    int ALU_EXTENT_X     = (params.out_width + read_unit - 1) / read_unit;
    auto LG2ALU_EXTENT_X = static_cast<int>(std::ceil(std::log(ALU_EXTENT_X) / std::log(2)));
    int ALU_EXTENT_Y     = (GRP_SZ >> LG2ALU_EXTENT_X);
    auto LG2ALU_EXTENT_Y = static_cast<int>(std::ceil(std::log(ALU_EXTENT_Y) / std::log(2)));

    // the wave is logical is a unit of shareing weights in SGPRs
    // it cannot be less than HW_WAVE_SIZE = 64
    // it cannot be larger than the group size.

    int LG2_WAVE_SZ0    = std::ceil(std::log(ALU_EXTENT_X) / std::log(2));
    int logical_wave_sz = std::max(1, ALU_EXTENT_X / hw_wave_sz) * hw_wave_sz;
    if(logical_wave_sz > GRP_SZ)
    {
        MIOPEN_LOG_E("logical_wave_sz > GRP_SZ");
        return ConvSolution(static_cast<miopenStatus_t>(-1));
    }
    int logical_n_waves = std::max(1, GRP_SZ / logical_wave_sz);
    int LG2_WAVE_SZ     = std::ceil(std::log(logical_wave_sz) / std::log(2));
    int WAVE_SZ1        = (logical_wave_sz >> LG2_WAVE_SZ0);
    int lg2_n_waves     = std::ceil(std::log(logical_n_waves) / std::log(2));
    int N_WAVES_MASK    = (1 << lg2_n_waves) - 1;

    int OUT_EXTENT1 = result.out_pix_tile1 * WAVE_SZ1;
    int OUT_EXTENT0 = (result.out_pix_tile0 << LG2_WAVE_SZ0);

    int total_out_maps = result.n_out_pix_tiles * logical_n_waves;

    // number of batches inside wk_item
    result.n_stacks = std::min(params.batch_sz, result.n_stacks);

    int N_HEIGHT_EXTENTS = (params.out_height + OUT_EXTENT1 - 1) / OUT_EXTENT1;
    int N_WIDTH_EXTENTS  = (params.out_width + OUT_EXTENT0 - 1) / OUT_EXTENT0;
    int N_GROUPS_PER_MAP = N_HEIGHT_EXTENTS * N_WIDTH_EXTENTS;

    result.grp_tile0 = GRP_SZ;
    result.grp_tile1 = 1;
    int grp_tile2    = 1;
    // auto in_tile0 = OUT_EXTENT0;
    // auto in_tile1 = OUT_EXTENT1;
    auto n_in_data_tiles = 1;

    //	_gen_comp_options += std::string(" -limit-vector-registers=64 ");

    KernelInfo construction_parameters;

    construction_parameters.comp_options =
        std::string(" -DMLO_DIR_FORWARD=") + (params.direction.IsForward() ? "1" : "0") +
        std::string(" -DMLO_GRP_SZ=") + std::to_string(static_cast<long long>(GRP_SZ)) +
        std::string(" -DMLO_GRP_SZ0=") + std::to_string(static_cast<long long>(result.grp_tile0)) +
        std::string(" -DMLO_GRP_SZ1=") + std::to_string(static_cast<long long>(result.grp_tile1)) +
        std::string(" -DMLO_GRP_SZ2=") + std::to_string(static_cast<long long>(grp_tile2)) +
        std::string(" -DMLO_FILTER_SIZE0=") +
        std::to_string(static_cast<long long>(params.kernel_size_w)) +
        std::string(" -DMLO_FILTER_SIZE1=") +
        std::to_string(static_cast<long long>(params.kernel_size_h)) +
        std::string(" -DMLO_FILTER_PAD0=") + std::to_string(static_cast<long long>(params.pad_w)) +
        std::string(" -DMLO_FILTER_PAD1=") + std::to_string(static_cast<long long>(params.pad_h)) +
        std::string(" -DMLO_N_OUTPUTS=") +
        std::to_string(static_cast<long long>(params.n_outputs)) + std::string(" -DMLO_N_INPUTS=") +
        std::to_string(static_cast<long long>(params.n_inputs)) + std::string(" -DMLO_BATCH_SZ=") +
        std::to_string(static_cast<long long>(params.batch_sz)) +
        std::string(" -DMLO_OUT_BATCH_STRIDE=") +
        std::to_string(static_cast<long long>(params.out_batch_stride)) +
        std::string(" -DMLO_OUT_CHANNEL_STRIDE=") +
        std::to_string(static_cast<long long>(params.out_channel_stride)) +
        std::string(" -DMLO_OUT_STRIDE=") +
        std::to_string(static_cast<long long>(params.out_stride)) +
        std::string(" -DMLO_IN_BATCH_STRIDE=") +
        std::to_string(static_cast<long long>(params.in_batch_stride)) +
        std::string(" -DMLO_IN_CHANNEL_STRIDE=") +
        std::to_string(static_cast<long long>(params.in_channel_stride)) +
        std::string(" -DMLO_IN_STRIDE=") +
        std::to_string(static_cast<long long>(params.in_stride)) +
        std::string(" -DMLO_WEI_BATCH_STRIDE=") +
        std::to_string(static_cast<long long>(wei_bstride)) +
        std::string(" -DMLO_WEI_CHANNEL_STRIDE=") +
        std::to_string(static_cast<long long>(wei_cstride)) + std::string(" -DMLO_IN_WIDTH=") +
        std::to_string(static_cast<long long>(params.in_width)) + std::string(" -DMLO_IN_HEIGHT=") +
        std::to_string(static_cast<long long>(params.in_height)) +
        std::string(" -DMLO_N_LCL_BATCHS=") +
        std::to_string(static_cast<long long>(result.n_stacks)) // # of diff stacks (part of batch).
        + std::string(" -DMLO_N_LCL_OUT_MAPS=") +
        std::to_string(static_cast<long long>(
            result.n_out_pix_tiles)) // # output pixel tiles per wk-item (ALU)
        + std::string(" -DMLO_N_LCL_IN_MAPS=") +
        std::to_string(
            static_cast<long long>(n_in_data_tiles)) // total # of blocks of different inputs in LDS
        + std::string(" -DMLO_OUT_TILE0=") +
        std::to_string(
            static_cast<long long>(result.out_pix_tile0)) // size of ouptput tile per wk-item (ALU))
        + std::string(" -DMLO_OUT_TILE1=") +
        std::to_string(static_cast<long long>(result.out_pix_tile1)) //
        + std::string(" -DMLO_ALU_EXTENT_X=") +
        std::to_string(static_cast<long long>(ALU_EXTENT_X)) +
        std::string(" -DMLO_LG2ALU_EXTENT_X=") +
        std::to_string(static_cast<long long>(LG2ALU_EXTENT_X)) +
        std::string(" -DMLO_ALU_EXTENT_Y=") + std::to_string(static_cast<long long>(ALU_EXTENT_Y)) +
        std::string(" -DMLO_LG2ALU_EXTENT_Y=") +
        std::to_string(static_cast<long long>(LG2ALU_EXTENT_Y)) +
        std::string(" -DMLO_OUT_EXTENT1=") + std::to_string(static_cast<long long>(OUT_EXTENT1)) +
        std::string(" -DMLO_OUT_EXTENT0=") + std::to_string(static_cast<long long>(OUT_EXTENT0)) +
        std::string(" -DMLO_N_WAVES=") + std::to_string(static_cast<long long>(logical_n_waves)) +
        std::string(" -DMLO_N_WAVES_MASK=") + std::to_string(static_cast<long long>(N_WAVES_MASK)) +
        std::string(" -DMLO_LG2_WAVE_SZ=") + std::to_string(static_cast<long long>(LG2_WAVE_SZ)) +
        std::string(" -DMLO_LG2_WAVE_SZ0=") + std::to_string(static_cast<long long>(LG2_WAVE_SZ0)) +
        std::string(" -DMLO_READ_TYPE=") + READ_TYPE + std::string(" -DMLO_READ_UNIT=") +
        std::to_string(static_cast<long long>(read_unit)) + std::string(" -DMLO_CONV_BIAS=") +
        std::to_string(static_cast<long long>(params.bias)) + params.general_compile_options;

    std::cout << " MIOpenConvD3x3 " << construction_parameters.comp_options << std::endl;
    construction_parameters.l_wk.push_back(result.grp_tile0);
    construction_parameters.l_wk.push_back(result.grp_tile1);
    construction_parameters.l_wk.push_back(grp_tile2);

    size_t gbl_wk0 = N_GROUPS_PER_MAP;

    size_t gbl_wk1 = (params.n_outputs + total_out_maps - 1) / total_out_maps;
    size_t gbl_wk2 = (params.batch_sz + result.n_stacks - 1) / result.n_stacks;

    construction_parameters.g_wk.push_back(gbl_wk0 * result.grp_tile0);
    construction_parameters.g_wk.push_back(gbl_wk1);
    construction_parameters.g_wk.push_back(gbl_wk2);

    construction_parameters.kernel_file = "MIOpenConvD3x3.cl";
    construction_parameters.kernel_name = "MIOpenCvD3x3_WSR0";

    result.construction_params.push_back(construction_parameters);
    return result;
}
} // namespace solver
} // namespace miopen
<commit_msg>removed cout (#2009)<commit_after>/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#include "miopen/solver.hpp"
#include "miopen/handle.hpp"

namespace miopen {
namespace solver {

bool ConvOclDirectFwd3x3::IsApplicable(const ConvolutionContext& params) const
{
    if(!params.Is2d())
        return false;
    if(!(params.IsFp32() || params.IsFp16() || params.IsBfp16()))
        return false;

    return params.kernel_size_w == 3 && params.kernel_size_h == 3 && params.pad_w == 1 &&
           params.pad_h == 1 && params.kernel_stride_w == 1 && params.kernel_stride_h == 1 &&
           params.kernel_dilation_w == 1 && params.kernel_dilation_h == 1 &&
           params.group_counts == 1 && params.direction.IsForward() &&
           (params.out_width == 512 || params.out_width == 64 || params.out_width == 128 ||
            params.out_width == 256);
}

ConvSolution ConvOclDirectFwd3x3::GetSolution(const ConvolutionContext& params) const
{
    ConvSolution result;
    // size_t localMemSize = params.stream.GetLocalMemorySize();
    auto hw_wave_sz = 64;
    // auto dev_local_mem_sz = localMemSize; // in bytes
    int n_waves = 4;

    int wei_cstride = params.kernel_size_w * params.kernel_size_h;
    int wei_bstride = params.n_inputs * wei_cstride;

    result.out_pix_tile0   = 4;
    result.out_pix_tile1   = 2;
    result.n_stacks        = 1;
    result.n_out_pix_tiles = 4;
    int read_unit          = result.out_pix_tile0;
    //	std::string READ_TYPE = (read_unit == 1) ? "_FLOAT" : "_FLOAT" +
    // std::to_string(static_cast<long long>(read_unit));
    // MD: read_unit is never == 1
    std::string READ_TYPE = "_FLOAT" + std::to_string(static_cast<long long>(read_unit));

    int GRP_SZ = hw_wave_sz * n_waves;

    int ALU_EXTENT_X     = (params.out_width + read_unit - 1) / read_unit;
    auto LG2ALU_EXTENT_X = static_cast<int>(std::ceil(std::log(ALU_EXTENT_X) / std::log(2)));
    int ALU_EXTENT_Y     = (GRP_SZ >> LG2ALU_EXTENT_X);
    auto LG2ALU_EXTENT_Y = static_cast<int>(std::ceil(std::log(ALU_EXTENT_Y) / std::log(2)));

    // the wave is logical is a unit of shareing weights in SGPRs
    // it cannot be less than HW_WAVE_SIZE = 64
    // it cannot be larger than the group size.

    int LG2_WAVE_SZ0    = std::ceil(std::log(ALU_EXTENT_X) / std::log(2));
    int logical_wave_sz = std::max(1, ALU_EXTENT_X / hw_wave_sz) * hw_wave_sz;
    if(logical_wave_sz > GRP_SZ)
    {
        MIOPEN_LOG_E("logical_wave_sz > GRP_SZ");
        return ConvSolution(static_cast<miopenStatus_t>(-1));
    }
    int logical_n_waves = std::max(1, GRP_SZ / logical_wave_sz);
    int LG2_WAVE_SZ     = std::ceil(std::log(logical_wave_sz) / std::log(2));
    int WAVE_SZ1        = (logical_wave_sz >> LG2_WAVE_SZ0);
    int lg2_n_waves     = std::ceil(std::log(logical_n_waves) / std::log(2));
    int N_WAVES_MASK    = (1 << lg2_n_waves) - 1;

    int OUT_EXTENT1 = result.out_pix_tile1 * WAVE_SZ1;
    int OUT_EXTENT0 = (result.out_pix_tile0 << LG2_WAVE_SZ0);

    int total_out_maps = result.n_out_pix_tiles * logical_n_waves;

    // number of batches inside wk_item
    result.n_stacks = std::min(params.batch_sz, result.n_stacks);

    int N_HEIGHT_EXTENTS = (params.out_height + OUT_EXTENT1 - 1) / OUT_EXTENT1;
    int N_WIDTH_EXTENTS  = (params.out_width + OUT_EXTENT0 - 1) / OUT_EXTENT0;
    int N_GROUPS_PER_MAP = N_HEIGHT_EXTENTS * N_WIDTH_EXTENTS;

    result.grp_tile0 = GRP_SZ;
    result.grp_tile1 = 1;
    int grp_tile2    = 1;
    // auto in_tile0 = OUT_EXTENT0;
    // auto in_tile1 = OUT_EXTENT1;
    auto n_in_data_tiles = 1;

    //	_gen_comp_options += std::string(" -limit-vector-registers=64 ");

    KernelInfo construction_parameters;

    construction_parameters.comp_options =
        std::string(" -DMLO_DIR_FORWARD=") + (params.direction.IsForward() ? "1" : "0") +
        std::string(" -DMLO_GRP_SZ=") + std::to_string(static_cast<long long>(GRP_SZ)) +
        std::string(" -DMLO_GRP_SZ0=") + std::to_string(static_cast<long long>(result.grp_tile0)) +
        std::string(" -DMLO_GRP_SZ1=") + std::to_string(static_cast<long long>(result.grp_tile1)) +
        std::string(" -DMLO_GRP_SZ2=") + std::to_string(static_cast<long long>(grp_tile2)) +
        std::string(" -DMLO_FILTER_SIZE0=") +
        std::to_string(static_cast<long long>(params.kernel_size_w)) +
        std::string(" -DMLO_FILTER_SIZE1=") +
        std::to_string(static_cast<long long>(params.kernel_size_h)) +
        std::string(" -DMLO_FILTER_PAD0=") + std::to_string(static_cast<long long>(params.pad_w)) +
        std::string(" -DMLO_FILTER_PAD1=") + std::to_string(static_cast<long long>(params.pad_h)) +
        std::string(" -DMLO_N_OUTPUTS=") +
        std::to_string(static_cast<long long>(params.n_outputs)) + std::string(" -DMLO_N_INPUTS=") +
        std::to_string(static_cast<long long>(params.n_inputs)) + std::string(" -DMLO_BATCH_SZ=") +
        std::to_string(static_cast<long long>(params.batch_sz)) +
        std::string(" -DMLO_OUT_BATCH_STRIDE=") +
        std::to_string(static_cast<long long>(params.out_batch_stride)) +
        std::string(" -DMLO_OUT_CHANNEL_STRIDE=") +
        std::to_string(static_cast<long long>(params.out_channel_stride)) +
        std::string(" -DMLO_OUT_STRIDE=") +
        std::to_string(static_cast<long long>(params.out_stride)) +
        std::string(" -DMLO_IN_BATCH_STRIDE=") +
        std::to_string(static_cast<long long>(params.in_batch_stride)) +
        std::string(" -DMLO_IN_CHANNEL_STRIDE=") +
        std::to_string(static_cast<long long>(params.in_channel_stride)) +
        std::string(" -DMLO_IN_STRIDE=") +
        std::to_string(static_cast<long long>(params.in_stride)) +
        std::string(" -DMLO_WEI_BATCH_STRIDE=") +
        std::to_string(static_cast<long long>(wei_bstride)) +
        std::string(" -DMLO_WEI_CHANNEL_STRIDE=") +
        std::to_string(static_cast<long long>(wei_cstride)) + std::string(" -DMLO_IN_WIDTH=") +
        std::to_string(static_cast<long long>(params.in_width)) + std::string(" -DMLO_IN_HEIGHT=") +
        std::to_string(static_cast<long long>(params.in_height)) +
        std::string(" -DMLO_N_LCL_BATCHS=") +
        std::to_string(static_cast<long long>(result.n_stacks)) // # of diff stacks (part of batch).
        + std::string(" -DMLO_N_LCL_OUT_MAPS=") +
        std::to_string(static_cast<long long>(
            result.n_out_pix_tiles)) // # output pixel tiles per wk-item (ALU)
        + std::string(" -DMLO_N_LCL_IN_MAPS=") +
        std::to_string(
            static_cast<long long>(n_in_data_tiles)) // total # of blocks of different inputs in LDS
        + std::string(" -DMLO_OUT_TILE0=") +
        std::to_string(
            static_cast<long long>(result.out_pix_tile0)) // size of ouptput tile per wk-item (ALU))
        + std::string(" -DMLO_OUT_TILE1=") +
        std::to_string(static_cast<long long>(result.out_pix_tile1)) //
        + std::string(" -DMLO_ALU_EXTENT_X=") +
        std::to_string(static_cast<long long>(ALU_EXTENT_X)) +
        std::string(" -DMLO_LG2ALU_EXTENT_X=") +
        std::to_string(static_cast<long long>(LG2ALU_EXTENT_X)) +
        std::string(" -DMLO_ALU_EXTENT_Y=") + std::to_string(static_cast<long long>(ALU_EXTENT_Y)) +
        std::string(" -DMLO_LG2ALU_EXTENT_Y=") +
        std::to_string(static_cast<long long>(LG2ALU_EXTENT_Y)) +
        std::string(" -DMLO_OUT_EXTENT1=") + std::to_string(static_cast<long long>(OUT_EXTENT1)) +
        std::string(" -DMLO_OUT_EXTENT0=") + std::to_string(static_cast<long long>(OUT_EXTENT0)) +
        std::string(" -DMLO_N_WAVES=") + std::to_string(static_cast<long long>(logical_n_waves)) +
        std::string(" -DMLO_N_WAVES_MASK=") + std::to_string(static_cast<long long>(N_WAVES_MASK)) +
        std::string(" -DMLO_LG2_WAVE_SZ=") + std::to_string(static_cast<long long>(LG2_WAVE_SZ)) +
        std::string(" -DMLO_LG2_WAVE_SZ0=") + std::to_string(static_cast<long long>(LG2_WAVE_SZ0)) +
        std::string(" -DMLO_READ_TYPE=") + READ_TYPE + std::string(" -DMLO_READ_UNIT=") +
        std::to_string(static_cast<long long>(read_unit)) + std::string(" -DMLO_CONV_BIAS=") +
        std::to_string(static_cast<long long>(params.bias)) + params.general_compile_options;

    construction_parameters.l_wk.push_back(result.grp_tile0);
    construction_parameters.l_wk.push_back(result.grp_tile1);
    construction_parameters.l_wk.push_back(grp_tile2);

    size_t gbl_wk0 = N_GROUPS_PER_MAP;

    size_t gbl_wk1 = (params.n_outputs + total_out_maps - 1) / total_out_maps;
    size_t gbl_wk2 = (params.batch_sz + result.n_stacks - 1) / result.n_stacks;

    construction_parameters.g_wk.push_back(gbl_wk0 * result.grp_tile0);
    construction_parameters.g_wk.push_back(gbl_wk1);
    construction_parameters.g_wk.push_back(gbl_wk2);

    construction_parameters.kernel_file = "MIOpenConvD3x3.cl";
    construction_parameters.kernel_name = "MIOpenCvD3x3_WSR0";

    result.construction_params.push_back(construction_parameters);
    return result;
}
} // namespace solver
} // namespace miopen
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*!
 * \file tvm/target/compilation_config.cc
 * \brief Implementation of \p CompilationConfig for collecting \p Targets.
 */

#include <tvm/runtime/device_api.h>
#include <tvm/target/compilation_config.h>

namespace tvm {

TVM_REGISTER_NODE_TYPE(CompilationConfigNode);

void CompilationConfigNode::VisitAttrs(AttrVisitor* v) {
  v->Visit("legacy_target_map", &legacy_target_map);
  v->Visit("host_target", &host_target);
  v->Visit("primitive_targets", &primitive_targets);
  v->Visit("default_primitive_se_scope", &default_primitive_se_scope);
  v->Visit("host_se_scope", &host_se_scope);
  v->Visit("optional_homogenous_target", &optional_homogeneous_target);
  // NOTE: The se_scope_cache_ is not accessible via FFI.
}

SEScope CompilationConfigNode::CanonicalSEScope(const SEScope& se_scope) const {
  if (se_scope->target.defined()) {
    return se_scope_cache_.Unique(se_scope);
  }
  DLDeviceType device_type = se_scope->device_type();
  // TODO(mbs): Proper diagnostics.
  CHECK(device_type != kInvalidDeviceType)
      << "SEScope annotations must include at least a device_type";
  Target target = FindPrimitiveTargetOrFail(se_scope->device_type());
  return se_scope_cache_.Unique(
      SEScope(device_type, se_scope->virtual_device_id, target, se_scope->memory_scope));
}

void CompilationConfigNode::EstablishDefaultSEScopes(const transform::PassContext& pass_ctx) {
  //
  // Gather the hints as to what our default device type for the 'host' should be, and
  // create an appropriate target if we don't already have one.
  //
  DLDeviceType host_device_type;
  if (host_target.defined()) {
    CHECK(!host_target->host.defined()) << "Host targets are not expected to have hosts";
    host_device_type = static_cast<DLDeviceType>(host_target->kind->device_type);
    LOG(INFO) << "Using the given host target " << host_target->ToDebugString()
              << " of device type " << host_device_type << " for the host target";
    for (const auto& primitive_target : primitive_targets) {
      if (primitive_target->host.defined() &&
          !StructuralEqual()(primitive_target->host, host_target)) {
        LOG(WARNING) << "The primitive target " << primitive_target->ToDebugString()
                     << " already has a host which disagrees with the desired host target. It "
                        "will be ignored.";
      }
    }
  } else if (primitive_targets.size() == 1 && primitive_targets.front()->host.defined()) {
    host_target = primitive_targets.front()->GetHost().value();
    CHECK(!host_target->host.defined()) << "Host targets are not expected to have hosts";
    host_device_type = static_cast<DLDeviceType>(host_target->kind->device_type);
    LOG(INFO) << "Using the host of the unique primitive target, namely "
              << host_target->ToDebugString() << " of device type " << host_device_type
              << " for the host target";
  } else if (primitive_targets.size() == 1 &&
             primitive_targets.front()->kind->device_type == kDLCPU) {
    // In the homogenous case without an explicit host target just use the given target so long as
    // it's a CPU.
    host_device_type = kDLCPU;
    host_target = primitive_targets.front();
    LOG(INFO) << "Using the unique primitive target " << host_target->ToDebugString()
              << " of device type " << host_device_type << " for the host target";
  } else {
    // Fallback.
    host_device_type = kDLCPU;
    // Even if the list of available targets already includes one for kDLCPU we won't use it
    // in the hetrogeneous case since its options may not be appropriate for host code
    // (eg shape functions). Instead, create a fresh default Target.
    host_target = MakeDefaultTarget(host_device_type);
    LOG(WARNING) << "Using the default target " << host_target->ToDebugString()
                 << " of device type " << host_device_type << " for the host target";
  }
  ICHECK(host_target.defined());
  ICHECK(!host_target->host.defined());

  if (host_device_type != kDLCPU) {
    // I think we're on thin ice here until we've audited the code base for assumed kDLCPU.
    LOG(WARNING) << "The host target is not a CPU.";
  }

  //
  // Establish the host SEScope.
  //
  host_se_scope = se_scope_cache_.Unique(SEScope(host_device_type,
                                                 /*virtual_device_id=*/0, host_target));

  //
  // Now that we've settled on a host, make sure all the primitive Targets agree on it for
  // their 'host' field. This mutates the primitives.
  //
  Array<Target> new_primitve_targets;
  new_primitve_targets.reserve(primitive_targets.size());
  for (const auto& primitive_target : primitive_targets) {
    new_primitve_targets.push_back(Target(primitive_target, host_target));
  }
  primitive_targets = new_primitve_targets;

  //
  // Gather the hints as to what our default device type for primitives should be.
  //
  DLDeviceType default_primitive_device_type;
  Optional<Integer> opt_fallback_dev = pass_ctx->GetConfig<Integer>("relay.fallback_device_type");
  if (opt_fallback_dev) {
    const int64_t v = opt_fallback_dev.value()->value;
    CHECK_GT(v, 0)
        << "The 'relay.fallback_device_type' pass attribute is set to an invalid device type " << v;
    default_primitive_device_type = static_cast<DLDeviceType>(v);
    LOG(INFO) << "Using the 'relay.fallback_device_type' pass attribute "
              << default_primitive_device_type
              << " as the default device type for all primitive operations";
  } else if (primitive_targets.size() == 1) {
    // In the homogeneous case there's no free choice.
    default_primitive_device_type =
        static_cast<DLDeviceType>(primitive_targets.front()->kind->device_type);
    LOG(INFO) << "Using the device type " << default_primitive_device_type
              << " of the unique primitive target as the default device type for all primitive "
                 "operations";
  } else {
    // Fallback. Note that we'll require a primitive Target of kDLCPU device_type to be given
    // and won't manufacture one out of thin air.
    default_primitive_device_type = kDLCPU;
    LOG(WARNING) << "Using " << default_primitive_device_type
                 << " as the default device type for all primitive operations";
  }

  //
  // Establish the default primitive SEScope, choosing a known Target to match the device type.
  //
  default_primitive_se_scope = se_scope_cache_.Unique(
      SEScope(default_primitive_device_type,
              /*virtual_device_id=*/0, FindPrimitiveTargetOrFail(default_primitive_device_type)));
}

/* static */ Target CompilationConfigNode::MakeDefaultTarget(DLDeviceType device_type) {
  std::string name = runtime::DeviceName(device_type);
  if (name == "cpu") {
    if (runtime::Registry::Get("codegen.LLVMModuleCreate")) {
      // LLVM is available.
      return Target("llvm");
    } else {
      // LLVM is not available.
      // TODO(mbs): Already deprecated?
      return Target("stackvm");
    }
  } else {
    return Target(name);
  }
}

Target CompilationConfigNode::FindPrimitiveTargetOrFail(DLDeviceType device_type) const {
  auto itr = std::find_if(
      primitive_targets.begin(), primitive_targets.end(),
      [device_type](const Target& target) { return target->kind->device_type == device_type; });
  CHECK(itr != primitive_targets.end()) << "No target for device type " << device_type << " in the "
                                        << primitive_targets.size() << " given by the targets list";
  return *itr;
}

CompilationConfig::CompilationConfig(const transform::PassContext& pass_ctx,
                                     TargetMap legacy_target_map_arg,
                                     Target optional_host_target_arg) {
  VLOG_CONTEXT << "CompilationConfig";

  auto node = make_object<CompilationConfigNode>();

  for (const auto& pair : legacy_target_map_arg) {
    VLOG(0) << "Available primitive target " << pair.first << " = " << pair.second->ToDebugString();
  }
  if (optional_host_target_arg.defined()) {
    VLOG(0) << "Available host target " << optional_host_target_arg->ToDebugString();
  }

  // Capture the arguments in our representation.
  for (const auto& pair : legacy_target_map_arg) {
    node->primitive_targets.push_back(pair.second);
  }
  node->host_target = optional_host_target_arg;

  // Complete the targets vector and establish default scopes. After this primitive_targets will
  // contain the definitive list of all required targets, target_host will be defined, and
  // all primitive targets will have host target_host.
  node->EstablishDefaultSEScopes(pass_ctx);

  // LEGACY: Reconstruct the target map with all the primitive targets.
  for (const auto& primitive_target : node->primitive_targets) {
    node->legacy_target_map.Set(Integer(primitive_target->kind->device_type), primitive_target);
  }

  ICHECK(node->default_primitive_se_scope->target.defined());
  ICHECK(node->host_se_scope->target.defined());
  ICHECK_GT(node->primitive_targets.size(), 0U);

  // Legacy: Some passes only support homogenous compilation and expect the target to be
  // given by the global target context. Make this easy to detect.
  node->optional_homogeneous_target =
      node->primitive_targets.size() == 1 ? *node->primitive_targets.begin() : Target();

  for (const auto& target : node->primitive_targets) {
    LOG(INFO) << "Target " << target->ToDebugString() << " of device type "
              << target->kind->device_type << " is available for primitives";
  }
  LOG(INFO) << "Using default primitive scope " << node->default_primitive_se_scope;
  LOG(INFO) << "Using host scope " << node->host_se_scope;

  data_ = std::move(node);
}

TVM_REGISTER_GLOBAL("target.MakeCompilationConfig")
    .set_body_typed([](const transform::PassContext& pass_ctx, TargetMap legacy_target_map,
                       Target optional_host_target) -> CompilationConfig {
      return CompilationConfig(pass_ctx, std::move(legacy_target_map),
                               std::move(optional_host_target));
    });

}  // namespace tvm
<commit_msg>[HOTFIX][TARGET] Change LOG in compilation config to DLOG (#9486)<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*!
 * \file tvm/target/compilation_config.cc
 * \brief Implementation of \p CompilationConfig for collecting \p Targets.
 */

#include <tvm/runtime/device_api.h>
#include <tvm/target/compilation_config.h>

namespace tvm {

TVM_REGISTER_NODE_TYPE(CompilationConfigNode);

void CompilationConfigNode::VisitAttrs(AttrVisitor* v) {
  v->Visit("legacy_target_map", &legacy_target_map);
  v->Visit("host_target", &host_target);
  v->Visit("primitive_targets", &primitive_targets);
  v->Visit("default_primitive_se_scope", &default_primitive_se_scope);
  v->Visit("host_se_scope", &host_se_scope);
  v->Visit("optional_homogenous_target", &optional_homogeneous_target);
  // NOTE: The se_scope_cache_ is not accessible via FFI.
}

SEScope CompilationConfigNode::CanonicalSEScope(const SEScope& se_scope) const {
  if (se_scope->target.defined()) {
    return se_scope_cache_.Unique(se_scope);
  }
  DLDeviceType device_type = se_scope->device_type();
  // TODO(mbs): Proper diagnostics.
  CHECK(device_type != kInvalidDeviceType)
      << "SEScope annotations must include at least a device_type";
  Target target = FindPrimitiveTargetOrFail(se_scope->device_type());
  return se_scope_cache_.Unique(
      SEScope(device_type, se_scope->virtual_device_id, target, se_scope->memory_scope));
}

void CompilationConfigNode::EstablishDefaultSEScopes(const transform::PassContext& pass_ctx) {
  //
  // Gather the hints as to what our default device type for the 'host' should be, and
  // create an appropriate target if we don't already have one.
  //
  DLDeviceType host_device_type;
  if (host_target.defined()) {
    CHECK(!host_target->host.defined()) << "Host targets are not expected to have hosts";
    host_device_type = static_cast<DLDeviceType>(host_target->kind->device_type);
    DLOG(INFO) << "Using the given host target " << host_target->ToDebugString()
               << " of device type " << host_device_type << " for the host target";
    for (const auto& primitive_target : primitive_targets) {
      if (primitive_target->host.defined() &&
          !StructuralEqual()(primitive_target->host, host_target)) {
        DLOG(WARNING) << "The primitive target " << primitive_target->ToDebugString()
                      << " already has a host which disagrees with the desired host target. It "
                      << "will be ignored.";
      }
    }
  } else if (primitive_targets.size() == 1 && primitive_targets.front()->host.defined()) {
    host_target = primitive_targets.front()->GetHost().value();
    CHECK(!host_target->host.defined()) << "Host targets are not expected to have hosts";
    host_device_type = static_cast<DLDeviceType>(host_target->kind->device_type);
    DLOG(INFO) << "Using the host of the unique primitive target, namely "
               << host_target->ToDebugString() << " of device type " << host_device_type
               << " for the host target";
  } else if (primitive_targets.size() == 1 &&
             primitive_targets.front()->kind->device_type == kDLCPU) {
    // In the homogenous case without an explicit host target just use the given target so long as
    // it's a CPU.
    host_device_type = kDLCPU;
    host_target = primitive_targets.front();
    DLOG(INFO) << "Using the unique primitive target " << host_target->ToDebugString()
               << " of device type " << host_device_type << " for the host target";
  } else {
    // Fallback.
    host_device_type = kDLCPU;
    // Even if the list of available targets already includes one for kDLCPU we won't use it
    // in the hetrogeneous case since its options may not be appropriate for host code
    // (eg shape functions). Instead, create a fresh default Target.
    host_target = MakeDefaultTarget(host_device_type);
    DLOG(WARNING) << "Using the default target " << host_target->ToDebugString()
                  << " of device type " << host_device_type << " for the host target";
  }
  ICHECK(host_target.defined());
  ICHECK(!host_target->host.defined());

  if (host_device_type != kDLCPU) {
    // I think we're on thin ice here until we've audited the code base for assumed kDLCPU.
    LOG(WARNING) << "The host target is not a CPU.";
  }

  //
  // Establish the host SEScope.
  //
  host_se_scope = se_scope_cache_.Unique(SEScope(host_device_type,
                                                 /*virtual_device_id=*/0, host_target));

  //
  // Now that we've settled on a host, make sure all the primitive Targets agree on it for
  // their 'host' field. This mutates the primitives.
  //
  Array<Target> new_primitve_targets;
  new_primitve_targets.reserve(primitive_targets.size());
  for (const auto& primitive_target : primitive_targets) {
    new_primitve_targets.push_back(Target(primitive_target, host_target));
  }
  primitive_targets = new_primitve_targets;

  //
  // Gather the hints as to what our default device type for primitives should be.
  //
  DLDeviceType default_primitive_device_type;
  Optional<Integer> opt_fallback_dev = pass_ctx->GetConfig<Integer>("relay.fallback_device_type");
  if (opt_fallback_dev) {
    const int64_t v = opt_fallback_dev.value()->value;
    CHECK_GT(v, 0)
        << "The 'relay.fallback_device_type' pass attribute is set to an invalid device type " << v;
    default_primitive_device_type = static_cast<DLDeviceType>(v);
    DLOG(INFO) << "Using the 'relay.fallback_device_type' pass attribute "
               << default_primitive_device_type
               << " as the default device type for all primitive operations";
  } else if (primitive_targets.size() == 1) {
    // In the homogeneous case there's no free choice.
    default_primitive_device_type =
        static_cast<DLDeviceType>(primitive_targets.front()->kind->device_type);
    DLOG(INFO) << "Using the device type " << default_primitive_device_type
               << " of the unique primitive target as the default device type for all primitive "
               << "operations";
  } else {
    // Fallback. Note that we'll require a primitive Target of kDLCPU device_type to be given
    // and won't manufacture one out of thin air.
    default_primitive_device_type = kDLCPU;
    DLOG(WARNING) << "Using " << default_primitive_device_type
                  << " as the default device type for all primitive operations";
  }

  //
  // Establish the default primitive SEScope, choosing a known Target to match the device type.
  //
  default_primitive_se_scope = se_scope_cache_.Unique(
      SEScope(default_primitive_device_type,
              /*virtual_device_id=*/0, FindPrimitiveTargetOrFail(default_primitive_device_type)));
}

/* static */ Target CompilationConfigNode::MakeDefaultTarget(DLDeviceType device_type) {
  std::string name = runtime::DeviceName(device_type);
  if (name == "cpu") {
    if (runtime::Registry::Get("codegen.LLVMModuleCreate")) {
      // LLVM is available.
      return Target("llvm");
    } else {
      // LLVM is not available.
      // TODO(mbs): Already deprecated?
      return Target("stackvm");
    }
  } else {
    return Target(name);
  }
}

Target CompilationConfigNode::FindPrimitiveTargetOrFail(DLDeviceType device_type) const {
  auto itr = std::find_if(
      primitive_targets.begin(), primitive_targets.end(),
      [device_type](const Target& target) { return target->kind->device_type == device_type; });
  CHECK(itr != primitive_targets.end()) << "No target for device type " << device_type << " in the "
                                        << primitive_targets.size() << " given by the targets list";
  return *itr;
}

CompilationConfig::CompilationConfig(const transform::PassContext& pass_ctx,
                                     TargetMap legacy_target_map_arg,
                                     Target optional_host_target_arg) {
  VLOG_CONTEXT << "CompilationConfig";

  auto node = make_object<CompilationConfigNode>();

  for (const auto& pair : legacy_target_map_arg) {
    VLOG(0) << "Available primitive target " << pair.first << " = " << pair.second->ToDebugString();
  }
  if (optional_host_target_arg.defined()) {
    VLOG(0) << "Available host target " << optional_host_target_arg->ToDebugString();
  }

  // Capture the arguments in our representation.
  for (const auto& pair : legacy_target_map_arg) {
    node->primitive_targets.push_back(pair.second);
  }
  node->host_target = optional_host_target_arg;

  // Complete the targets vector and establish default scopes. After this primitive_targets will
  // contain the definitive list of all required targets, target_host will be defined, and
  // all primitive targets will have host target_host.
  node->EstablishDefaultSEScopes(pass_ctx);

  // LEGACY: Reconstruct the target map with all the primitive targets.
  for (const auto& primitive_target : node->primitive_targets) {
    node->legacy_target_map.Set(Integer(primitive_target->kind->device_type), primitive_target);
  }

  ICHECK(node->default_primitive_se_scope->target.defined());
  ICHECK(node->host_se_scope->target.defined());
  ICHECK_GT(node->primitive_targets.size(), 0U);

  // Legacy: Some passes only support homogenous compilation and expect the target to be
  // given by the global target context. Make this easy to detect.
  node->optional_homogeneous_target =
      node->primitive_targets.size() == 1 ? *node->primitive_targets.begin() : Target();

  for (const auto& target : node->primitive_targets) {
    DLOG(INFO) << "Target " << target->ToDebugString() << " of device type "
               << target->kind->device_type << " is available for primitives";
  }
  DLOG(INFO) << "Using default primitive scope " << node->default_primitive_se_scope;
  DLOG(INFO) << "Using host scope " << node->host_se_scope;

  data_ = std::move(node);
}

TVM_REGISTER_GLOBAL("target.MakeCompilationConfig")
    .set_body_typed([](const transform::PassContext& pass_ctx, TargetMap legacy_target_map,
                       Target optional_host_target) -> CompilationConfig {
      return CompilationConfig(pass_ctx, std::move(legacy_target_map),
                               std::move(optional_host_target));
    });

}  // namespace tvm
<|endoftext|>


<commit_before>// Copyright (c) 2013, Cloudera, inc.

#include <boost/lexical_cast.hpp>
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <vector>
#include "util/knapsack_solver.h"
#include "util/stopwatch.h"
#include "util/test_util.h"

using std::vector;

namespace kudu {

class TestKnapsack : public KuduTest {
};

// A simple test item for use with the knapsack solver.
// The real code will be solving knapsack over RowSet objects --
// using simple value/weight pairs in the tests makes it standalone.
struct TestItem {
  TestItem(double v, int w)
    : value(v), weight(w) {
  }

  double value;
  int weight;
};

// A traits class to adapt the knapsack solver to TestItem.
struct TestItemTraits {
  typedef TestItem item_type;
  typedef double value_type;
  static int get_weight(const TestItem &item) {
    return item.weight;
  }
  static value_type get_value(const TestItem &item) {
    return item.value;
  }
};

// Generate random items into the provided vector.
static void GenerateRandomItems(int n_items, int max_weight,
                                vector<TestItem> *out) {
  for (int i = 0; i < n_items; i++) {
    double value = 10000.0 / (random() % 10000 + 1);
    int weight = random() % max_weight;
    out->push_back(TestItem(value, weight));
  }
}

// Join and stringify the given list of ints.
static string JoinInts(const vector<int> &ints) {
  string ret;
  for (int i = 0; i < ints.size(); i++) {
    if (i > 0) {
      ret.push_back(',');
    }
    ret.append(boost::lexical_cast<string>(ints[i]));
  }
  return ret;
}

TEST_F(TestKnapsack, Basics) {
  KnapsackSolver<TestItemTraits> solver;

  vector<TestItem> in;
  in.push_back(TestItem(500, 3));
  in.push_back(TestItem(110, 1));
  in.push_back(TestItem(125, 1));
  in.push_back(TestItem(100, 1));

  vector<int> out;
  double max_val;

  // For 1 weight, pick item 2
  solver.Solve(in, 1, &out, &max_val);
  ASSERT_DOUBLE_EQ(125, max_val);
  ASSERT_EQ("2", JoinInts(out));
  out.clear();

  // For 2 weight, pick item 1, 2
  solver.Solve(in, 2, &out, &max_val);
  ASSERT_DOUBLE_EQ(110 + 125, max_val);
  ASSERT_EQ("2,1", JoinInts(out));
  out.clear();

  // For 3 weight, pick item 0
  solver.Solve(in, 3, &out, &max_val);
  ASSERT_DOUBLE_EQ(500, max_val);
  ASSERT_EQ("0", JoinInts(out));
  out.clear();

  // For 10 weight, pick all.
  solver.Solve(in, 10, &out, &max_val);
  ASSERT_DOUBLE_EQ(500 + 110 + 125 + 100, max_val);
  ASSERT_EQ("3,2,1,0", JoinInts(out));
  out.clear();
}

// Test which generates random knapsack instances and verifies
// that the result satisfies the constraints.
TEST_F(TestKnapsack, Randomized) {
  SeedRandom();
  KnapsackSolver<TestItemTraits> solver;

  const int kNumTrials = AllowSlowTests() ? 200 : 1;
  const int kMaxWeight = 1000;
  const int kNumItems = 1000;

  for (int i = 0; i < kNumTrials; i++) {
    vector<TestItem> in;
    vector<int> out;
    GenerateRandomItems(kNumItems, kMaxWeight, &in);
    double max_val;
    int max_weight = random() % kMaxWeight;
    solver.Solve(in, max_weight, &out, &max_val);

    // Verify that the max_val is equal to the sum of the chosen items' values.
    double sum_val = 0;
    int sum_weight = 0;
    BOOST_FOREACH(int i, out) {
      sum_val += in[i].value;
      sum_weight += in[i].weight;
    }
    ASSERT_DOUBLE_EQ(max_val, sum_val);
    ASSERT_LE(sum_weight, max_weight);
  }
}

#ifdef NDEBUG
TEST_F(TestKnapsack, Benchmark) {
  KnapsackSolver<TestItemTraits> solver;

  const int kNumTrials = 1000;
  const int kMaxWeight = 1000;
  const int kNumItems = 1000;

  vector<TestItem> in;
  GenerateRandomItems(kNumItems, kMaxWeight, &in);

  LOG_TIMING(INFO, "benchmark") {
    vector<int> out;
    for (int i = 0; i < kNumTrials; i++) {
      out.clear();
      double max_val;
      solver.Solve(in, random() % kMaxWeight, &out, &max_val);
    }
  }
}
#endif

} // namespace kudu
<commit_msg>knapsack_solver-test: use ASSERT_NEAR instead of ASSERT_DOUBLE_EQ<commit_after>// Copyright (c) 2013, Cloudera, inc.

#include <boost/lexical_cast.hpp>
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <vector>
#include "util/knapsack_solver.h"
#include "util/stopwatch.h"
#include "util/test_util.h"

using std::vector;

namespace kudu {

class TestKnapsack : public KuduTest {
};

// A simple test item for use with the knapsack solver.
// The real code will be solving knapsack over RowSet objects --
// using simple value/weight pairs in the tests makes it standalone.
struct TestItem {
  TestItem(double v, int w)
    : value(v), weight(w) {
  }

  double value;
  int weight;
};

// A traits class to adapt the knapsack solver to TestItem.
struct TestItemTraits {
  typedef TestItem item_type;
  typedef double value_type;
  static int get_weight(const TestItem &item) {
    return item.weight;
  }
  static value_type get_value(const TestItem &item) {
    return item.value;
  }
};

// Generate random items into the provided vector.
static void GenerateRandomItems(int n_items, int max_weight,
                                vector<TestItem> *out) {
  for (int i = 0; i < n_items; i++) {
    double value = 10000.0 / (random() % 10000 + 1);
    int weight = random() % max_weight;
    out->push_back(TestItem(value, weight));
  }
}

// Join and stringify the given list of ints.
static string JoinInts(const vector<int> &ints) {
  string ret;
  for (int i = 0; i < ints.size(); i++) {
    if (i > 0) {
      ret.push_back(',');
    }
    ret.append(boost::lexical_cast<string>(ints[i]));
  }
  return ret;
}

TEST_F(TestKnapsack, Basics) {
  KnapsackSolver<TestItemTraits> solver;

  vector<TestItem> in;
  in.push_back(TestItem(500, 3));
  in.push_back(TestItem(110, 1));
  in.push_back(TestItem(125, 1));
  in.push_back(TestItem(100, 1));

  vector<int> out;
  double max_val;

  // For 1 weight, pick item 2
  solver.Solve(in, 1, &out, &max_val);
  ASSERT_DOUBLE_EQ(125, max_val);
  ASSERT_EQ("2", JoinInts(out));
  out.clear();

  // For 2 weight, pick item 1, 2
  solver.Solve(in, 2, &out, &max_val);
  ASSERT_DOUBLE_EQ(110 + 125, max_val);
  ASSERT_EQ("2,1", JoinInts(out));
  out.clear();

  // For 3 weight, pick item 0
  solver.Solve(in, 3, &out, &max_val);
  ASSERT_DOUBLE_EQ(500, max_val);
  ASSERT_EQ("0", JoinInts(out));
  out.clear();

  // For 10 weight, pick all.
  solver.Solve(in, 10, &out, &max_val);
  ASSERT_DOUBLE_EQ(500 + 110 + 125 + 100, max_val);
  ASSERT_EQ("3,2,1,0", JoinInts(out));
  out.clear();
}

// Test which generates random knapsack instances and verifies
// that the result satisfies the constraints.
TEST_F(TestKnapsack, Randomized) {
  SeedRandom();
  KnapsackSolver<TestItemTraits> solver;

  const int kNumTrials = AllowSlowTests() ? 200 : 1;
  const int kMaxWeight = 1000;
  const int kNumItems = 1000;

  for (int i = 0; i < kNumTrials; i++) {
    vector<TestItem> in;
    vector<int> out;
    GenerateRandomItems(kNumItems, kMaxWeight, &in);
    double max_val;
    int max_weight = random() % kMaxWeight;
    solver.Solve(in, max_weight, &out, &max_val);

    // Verify that the max_val is equal to the sum of the chosen items' values.
    double sum_val = 0;
    int sum_weight = 0;
    BOOST_FOREACH(int i, out) {
      sum_val += in[i].value;
      sum_weight += in[i].weight;
    }
    ASSERT_NEAR(max_val, sum_val, 0.000001);
    ASSERT_LE(sum_weight, max_weight);
  }
}

#ifdef NDEBUG
TEST_F(TestKnapsack, Benchmark) {
  KnapsackSolver<TestItemTraits> solver;

  const int kNumTrials = 1000;
  const int kMaxWeight = 1000;
  const int kNumItems = 1000;

  vector<TestItem> in;
  GenerateRandomItems(kNumItems, kMaxWeight, &in);

  LOG_TIMING(INFO, "benchmark") {
    vector<int> out;
    for (int i = 0; i < kNumTrials; i++) {
      out.clear();
      double max_val;
      solver.Solve(in, random() % kMaxWeight, &out, &max_val);
    }
  }
}
#endif

} // namespace kudu
<|endoftext|>


<commit_before>#include <xzero/testing.h>
#include <xzero/BufferUtil.h>
#include <xzero/raft/Generator.h>

using namespace xzero;
using namespace xzero::raft;

TEST(raft_Generator, VoteRequest) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateVoteRequest(VoteRequest{ .term = 0x11,
                                     .candidateId = 0x22,
                                     .lastLogIndex = 0x33,
                                     .lastLogTerm = 0x44 });
  EXPECT_EQ("\x01\x11\x22\x33\x44", out);
}

TEST(raft_Generator, VoteResponse) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateVoteResponse(VoteResponse{ .term = 0x11,
                                       .voteGranted = true });
  EXPECT_EQ("\x02\x11\x01", out);
}

TEST(raft_Generator, AppendEntriesRequest) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateAppendEntriesRequest(AppendEntriesRequest{
      .term = 0x11,
      .leaderId = 0x12,
      .prevLogIndex = 0x13,
      .prevLogTerm = 0x14,
      .leaderCommit = 0x15,
      .entries = { LogEntry{0x11, Command{1, 2, 3, 4}},
                   LogEntry{0x11, Command{5, 6, 7, 8}} }});
  EXPECT_EQ("\x03\x11\x12\x13\x14\x15\x02\x11\x01\x04\x01\x02\x03\x04\x11\x01\x04\x05\x06\x07\x08", out);
}

TEST(raft_Generator, AppendEntriesResponse) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateAppendEntriesResponse(AppendEntriesResponse{
      .term = 2,
      .success = true });

  EXPECT_EQ("\x04\x02\x01", out);
}

TEST(raft_Generator, InstallSnapshotRequest) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateInstallSnapshotRequest(InstallSnapshotRequest{
      .term = 1,
      .leaderId = 2,
      .lastIncludedIndex = 3,
      .lastIncludedTerm = 4,
      .offset = 0,
      .data = {0x42, 0x13, 0x37},
      .done = true });

  EXPECT_EQ("\x05\x01\x02\x03\x04\x00\x03\x42\x13\x37\x01", out);
}

TEST(raft_Generator, InstallSnapshotResponse) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateInstallSnapshotResponse(InstallSnapshotResponse{ .term = 1 });

  EXPECT_EQ("\x06\x01", out);
}
<commit_msg>[xzero] Generator: adds missing copyright header<commit_after>// This file is part of the "x0" project, http://github.com/christianparpart/x0>
//   (c) 2009-2017 Christian Parpart <christian@parpart.family>
//
// Licensed under the MIT License (the "License"); you may not use this
// file except in compliance with the License. You may obtain a copy of
// the License at: http://opensource.org/licenses/MIT
#include <xzero/testing.h>
#include <xzero/BufferUtil.h>
#include <xzero/raft/Generator.h>

using namespace xzero;
using namespace xzero::raft;

TEST(raft_Generator, VoteRequest) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateVoteRequest(VoteRequest{ .term = 0x11,
                                     .candidateId = 0x22,
                                     .lastLogIndex = 0x33,
                                     .lastLogTerm = 0x44 });
  EXPECT_EQ("\x01\x11\x22\x33\x44", out);
}

TEST(raft_Generator, VoteResponse) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateVoteResponse(VoteResponse{ .term = 0x11,
                                       .voteGranted = true });
  EXPECT_EQ("\x02\x11\x01", out);
}

TEST(raft_Generator, AppendEntriesRequest) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateAppendEntriesRequest(AppendEntriesRequest{
      .term = 0x11,
      .leaderId = 0x12,
      .prevLogIndex = 0x13,
      .prevLogTerm = 0x14,
      .leaderCommit = 0x15,
      .entries = { LogEntry{0x11, Command{1, 2, 3, 4}},
                   LogEntry{0x11, Command{5, 6, 7, 8}} }});
  EXPECT_EQ("\x03\x11\x12\x13\x14\x15\x02\x11\x01\x04\x01\x02\x03\x04\x11\x01\x04\x05\x06\x07\x08", out);
}

TEST(raft_Generator, AppendEntriesResponse) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateAppendEntriesResponse(AppendEntriesResponse{
      .term = 2,
      .success = true });

  EXPECT_EQ("\x04\x02\x01", out);
}

TEST(raft_Generator, InstallSnapshotRequest) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateInstallSnapshotRequest(InstallSnapshotRequest{
      .term = 1,
      .leaderId = 2,
      .lastIncludedIndex = 3,
      .lastIncludedTerm = 4,
      .offset = 0,
      .data = {0x42, 0x13, 0x37},
      .done = true });

  EXPECT_EQ("\x05\x01\x02\x03\x04\x00\x03\x42\x13\x37\x01", out);
}

TEST(raft_Generator, InstallSnapshotResponse) {
  Buffer out;
  raft::Generator g(BufferUtil::writer(&out));
  g.generateInstallSnapshotResponse(InstallSnapshotResponse{ .term = 1 });

  EXPECT_EQ("\x06\x01", out);
}
<|endoftext|>


<commit_before>#ifndef STAN_MATH_PRIM_SCAL_META_GET_HPP
#define STAN_MATH_PRIM_SCAL_META_GET_HPP

#include <stan/math/prim/arr/meta/get.hpp>

#include <cmath>
#include <cstddef>
#include <type_traits>

namespace stan {

template <typename T,
          typename = std::enable_if_t<std::is_fundamental<T>::value>>
inline T get(const T& x, size_t n) {
  return x;
}

}  // namespace stan
#endif
<commit_msg>remove conditional templating from get.hpp in scal.<commit_after>#ifndef STAN_MATH_PRIM_SCAL_META_GET_HPP
#define STAN_MATH_PRIM_SCAL_META_GET_HPP

#include <cmath>
#include <cstddef>
#include <type_traits>

namespace stan {

template <typename T>
inline T get(const T& x, size_t n) {
  return x;
}

}  // namespace stan
#endif
<|endoftext|>


<commit_before>/*
 * Copyright (c) [2017-2018] SUSE LLC
 *
 * All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, contact Novell, Inc.
 *
 * To contact Novell about this file by physical or electronic mail, you may
 * find current contact information at www.novell.com.
 */

#include "storage/Devices/MultipathImpl.h"
#include "storage/Devicegraph.h"
#include "storage/Storage.h"
#include "storage/Prober.h"
#include "storage/SystemInfo/SystemInfo.h"
#include "storage/Utils/Exception.h"
#include "storage/Utils/StorageTmpl.h"
#include "storage/Utils/StorageTypes.h"
#include "storage/Utils/StorageDefines.h"
#include "storage/Utils/XmlFile.h"
#include "storage/Utils/SystemCmd.h"
#include "storage/UsedFeatures.h"
#include "storage/Holders/User.h"
#include "storage/Utils/StorageTypes.h"
#include "storage/Utils/CallbacksImpl.h"

namespace storage
{

    using namespace std;

    const char* DeviceTraits<Multipath>::classname = "Multipath";

    Multipath::Impl::Impl(const string& name)
	: Partitionable::Impl(name), vendor(), model(), rotational(false)
    {
	if (!is_valid_name(name))
	    ST_THROW(Exception("invalid Multipath name"));

	set_dm_table_name(name.substr(strlen(DEVMAPPERDIR "/")));
    }

    Multipath::Impl::Impl(const string& name, const Region& region)
	: Partitionable::Impl(name, region), vendor(), model(), rotational(false)
    {
	if (!is_valid_name(name))
	    ST_THROW(Exception("invalid Multipath name"));

	set_dm_table_name(name.substr(strlen(DEVMAPPERDIR "/")));
    }

    Multipath::Impl::Impl(const xmlNode* node)
	: Partitionable::Impl(node), vendor(), model(), rotational(false)
    {
	getChildValue(node, "vendor", vendor);
	getChildValue(node, "model", model);

	getChildValue(node, "rotational", rotational);
    }

    string
    Multipath::Impl::get_pretty_classname() const
    {
	// TRANSLATORS: name of object
	return _("Multipath").translated;
    }

    void
    Multipath::Impl::check(const CheckCallbacks* check_callbacks) const
    {
	Partitionable::Impl::check(check_callbacks);

	if (!is_valid_name(get_name()))
	    ST_THROW(Exception("invalid name"));
    }

    bool
    Multipath::Impl::is_valid_name(const string& name)
    {
	return boost::starts_with(name, DEVMAPPERDIR "/");
    }

    bool
    Multipath::Impl::activate_multipaths(const ActivateCallbacks* activate_callbacks)
    {
	y2mil("activate_multipaths");

	try
	{
	    bool looks_like_real_multipath = CmdMultipath(true).looks_like_real_multipath();

	    if (!activate_callbacks->multipath(looks_like_real_multipath))
	    {
		y2mil("user canceled activation of multipath");
		return false;
	    }
	    else
	    {
		y2mil("user allowed activation of multipath");
	    }

	    // TRANSLATORS: progress message
	    message_callback(activate_callbacks, _("Activating multipath"));

	    try
	    {
		SystemCmd cmd1(MULTIPATHBIN, SystemCmd::DoThrow);

		SystemCmd(UDEVADMBIN_SETTLE);

		SystemCmd cmd2(MULTIPATHDBIN, SystemCmd::DoThrow);

		SystemCmd(UDEVADMBIN_SETTLE);

		return true;
	    }
	    catch (const Exception& exception)
	    {
		// TRANSLATORS: error message
		error_callback(activate_callbacks, _("Activating multipath failed"), exception);

		return false;
	    }
	}
	catch (const Exception& exception)
	{
	    ST_CAUGHT(exception);

	    if (typeid(exception) == typeid(Aborted))
		ST_RETHROW(exception);

	    // Ignore failure to detect whether multipath needs to be activated.

	    return false;
	}
    }

    bool
    Multipath::Impl::deactivate_multipaths()
    {
	y2mil("deactivate_multipaths");

	string cmd_line1 = MULTIPATHDBIN " -k'shutdown'";

	SystemCmd cmd1(cmd_line1);

	string cmd_line2 = MULTIPATHBIN " -F";

	SystemCmd cmd2(cmd_line2);

	return cmd2.retcode() == 0;
    }

    void
    Multipath::Impl::probe_multipaths(Prober& prober)
    {
	const CmdMultipath& cmd_multipath = prober.get_system_info().getCmdMultipath();

	for (const string& dm_name : cmd_multipath.get_entries())
	{
	    Multipath* multipath = Multipath::create(prober.get_system(), DEVMAPPERDIR "/" + dm_name);
	    multipath->get_impl().probe_pass_1a(prober);
	}
    }

    void
    Multipath::Impl::probe_pass_1a(Prober& prober)
    {
	Partitionable::Impl::probe_pass_1a(prober);

	const File rotational_file = prober.get_system_info().getFile(SYSFSDIR + get_sysfs_path() +
								      "/queue/rotational");
	rotational = rotational_file.get<bool>();

	const CmdMultipath& cmd_multipath = prober.get_system_info().getCmdMultipath();

	const CmdMultipath::Entry& entry = cmd_multipath.get_entry(get_dm_table_name());

	vendor = entry.vendor;
	model = entry.model;
    }

    void
    Multipath::Impl::probe_pass_1b(Prober& prober)
    {
	const CmdMultipath& cmd_multipath = prober.get_system_info().getCmdMultipath();

	const CmdMultipath::Entry& entry = cmd_multipath.get_entry(get_dm_table_name());

	for (const string& device : entry.devices)
	{
	    BlkDevice* blk_device = BlkDevice::Impl::find_by_any_name(prober.get_system(), device,
								      prober.get_system_info());
	    User::create(prober.get_system(), blk_device, get_non_impl());
	}
    }

    uint64_t
    Multipath::Impl::used_features() const
    {
	return UF_MULTIPATH | Partitionable::Impl::used_features();
    }

    void
    Multipath::Impl::save(xmlNode* node) const
    {
	Partitionable::Impl::save(node);

	setChildValue(node, "vendor", vendor);
	setChildValue(node, "model", model);

	setChildValueIf(node, "rotational", rotational, rotational);
    }

    void
    Multipath::Impl::add_create_actions(Actiongraph::Impl& actiongraph) const
    {
	ST_THROW(Exception("cannot create multipath"));
    }

    void
    Multipath::Impl::add_delete_actions(Actiongraph::Impl& actiongraph) const
    {
	ST_THROW(Exception("cannot delete multipath"));
    }

    bool
    Multipath::Impl::equal(const Device::Impl& rhs_base) const
    {
	const Impl& rhs = dynamic_cast<const Impl&>(rhs_base);

	if (!Partitionable::Impl::equal(rhs))
	    return false;

	return vendor == rhs.vendor && model == rhs.model && rotational == rhs.rotational;
    }

    void
    Multipath::Impl::log_diff(std::ostream& log, const Device::Impl& rhs_base) const
    {
	const Impl& rhs = dynamic_cast<const Impl&>(rhs_base);

	Partitionable::Impl::log_diff(log, rhs);

	storage::log_diff(log, "vendor", vendor, rhs.vendor);
	storage::log_diff(log, "model", model, rhs.model);

	storage::log_diff(log, "rotational", rotational, rhs.rotational);
    }

    void
    Multipath::Impl::print(std::ostream& out) const
    {
	Partitionable::Impl::print(out);

	out << " vendor:" << vendor << " model:" << model;

	if (rotational)
	    out << " rotational";
    }

    void
    Multipath::Impl::process_udev_ids(vector<string>& udev_ids) const
    {
	// See doc/udev.md.

	static const vector<string> allowed_prefixes = { "scsi-", "wwn-", "dm-name-", "dm-uuid-" };

	erase_if(udev_ids, [](const string& udev_id) {
	    return none_of(allowed_prefixes.begin(), allowed_prefixes.end(), [&udev_id](const string& prefix)
			   { return boost::starts_with(udev_id, prefix); });
	});

	stable_partition(udev_ids.begin(), udev_ids.end(), string_starts_with("wwn-"));
	stable_partition(udev_ids.begin(), udev_ids.end(), string_starts_with("scsi-"));
    }

    Text
    Multipath::Impl::do_deactivate_text(Tense tense) const
    {
	Text text = tenser(tense,
			   // TRANSLATORS: displayed before action,
			   // %1$s is replaced by multipath name (e.g. /dev/mapper/36005076305ffc73a00000000000013b4),
			   // %2$s is replaced by size (e.g. 1 TiB)
			   _("Deactivate multipath %1$s (%2$s)"),
			   // TRANSLATORS: displayed during action,
			   // %1$s is replaced by multipath name (e.g. /dev/mapper/36005076305ffc73a00000000000013b4),
			   // %2$s is replaced by size (e.g. 1 TiB)
			   _("Deactivating multipath %1$s (%2$s)"));

	return sformat(text, get_displayname().c_str(), get_size_string().c_str());
    }

    void
    Multipath::Impl::do_deactivate() const
    {
	string cmd_line = MULTIPATHBIN " -f " + quote(get_name());

	SystemCmd cmd(cmd_line, SystemCmd::DoThrow);
    }

}
<commit_msg>- improved logging<commit_after>/*
 * Copyright (c) [2017-2018] SUSE LLC
 *
 * All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of version 2 of the GNU General Public License as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, contact Novell, Inc.
 *
 * To contact Novell about this file by physical or electronic mail, you may
 * find current contact information at www.novell.com.
 */

#include "storage/Devices/MultipathImpl.h"
#include "storage/Devicegraph.h"
#include "storage/Storage.h"
#include "storage/Prober.h"
#include "storage/SystemInfo/SystemInfo.h"
#include "storage/Utils/Exception.h"
#include "storage/Utils/StorageTmpl.h"
#include "storage/Utils/StorageTypes.h"
#include "storage/Utils/StorageDefines.h"
#include "storage/Utils/XmlFile.h"
#include "storage/Utils/SystemCmd.h"
#include "storage/UsedFeatures.h"
#include "storage/Holders/User.h"
#include "storage/Utils/StorageTypes.h"
#include "storage/Utils/CallbacksImpl.h"

namespace storage
{

    using namespace std;

    const char* DeviceTraits<Multipath>::classname = "Multipath";

    Multipath::Impl::Impl(const string& name)
	: Partitionable::Impl(name), vendor(), model(), rotational(false)
    {
	if (!is_valid_name(name))
	    ST_THROW(Exception("invalid Multipath name"));

	set_dm_table_name(name.substr(strlen(DEVMAPPERDIR "/")));
    }

    Multipath::Impl::Impl(const string& name, const Region& region)
	: Partitionable::Impl(name, region), vendor(), model(), rotational(false)
    {
	if (!is_valid_name(name))
	    ST_THROW(Exception("invalid Multipath name"));

	set_dm_table_name(name.substr(strlen(DEVMAPPERDIR "/")));
    }

    Multipath::Impl::Impl(const xmlNode* node)
	: Partitionable::Impl(node), vendor(), model(), rotational(false)
    {
	getChildValue(node, "vendor", vendor);
	getChildValue(node, "model", model);

	getChildValue(node, "rotational", rotational);
    }

    string
    Multipath::Impl::get_pretty_classname() const
    {
	// TRANSLATORS: name of object
	return _("Multipath").translated;
    }

    void
    Multipath::Impl::check(const CheckCallbacks* check_callbacks) const
    {
	Partitionable::Impl::check(check_callbacks);

	if (!is_valid_name(get_name()))
	    ST_THROW(Exception("invalid name"));
    }

    bool
    Multipath::Impl::is_valid_name(const string& name)
    {
	return boost::starts_with(name, DEVMAPPERDIR "/");
    }

    bool
    Multipath::Impl::activate_multipaths(const ActivateCallbacks* activate_callbacks)
    {
	y2mil("activate_multipaths");

	try
	{
	    bool looks_like_real_multipath = CmdMultipath(true).looks_like_real_multipath();

	    y2mil("looks-like-real-multipath:" << looks_like_real_multipath);

	    if (!activate_callbacks->multipath(looks_like_real_multipath))
	    {
		y2mil("user does want multipath");
		return false;
	    }
	    else
	    {
		y2mil("user does not want multipath");
	    }

	    // TRANSLATORS: progress message
	    message_callback(activate_callbacks, _("Activating multipath"));

	    try
	    {
		SystemCmd cmd1(MULTIPATHBIN, SystemCmd::DoThrow);

		SystemCmd(UDEVADMBIN_SETTLE);

		SystemCmd cmd2(MULTIPATHDBIN, SystemCmd::DoThrow);

		SystemCmd(UDEVADMBIN_SETTLE);

		return true;
	    }
	    catch (const Exception& exception)
	    {
		// TRANSLATORS: error message
		error_callback(activate_callbacks, _("Activating multipath failed"), exception);

		return false;
	    }
	}
	catch (const Exception& exception)
	{
	    ST_CAUGHT(exception);

	    if (typeid(exception) == typeid(Aborted))
		ST_RETHROW(exception);

	    // Ignore failure to detect whether multipath needs to be activated.

	    return false;
	}
    }

    bool
    Multipath::Impl::deactivate_multipaths()
    {
	y2mil("deactivate_multipaths");

	string cmd_line1 = MULTIPATHDBIN " -k'shutdown'";

	SystemCmd cmd1(cmd_line1);

	string cmd_line2 = MULTIPATHBIN " -F";

	SystemCmd cmd2(cmd_line2);

	return cmd2.retcode() == 0;
    }

    void
    Multipath::Impl::probe_multipaths(Prober& prober)
    {
	const CmdMultipath& cmd_multipath = prober.get_system_info().getCmdMultipath();

	for (const string& dm_name : cmd_multipath.get_entries())
	{
	    Multipath* multipath = Multipath::create(prober.get_system(), DEVMAPPERDIR "/" + dm_name);
	    multipath->get_impl().probe_pass_1a(prober);
	}
    }

    void
    Multipath::Impl::probe_pass_1a(Prober& prober)
    {
	Partitionable::Impl::probe_pass_1a(prober);

	const File rotational_file = prober.get_system_info().getFile(SYSFSDIR + get_sysfs_path() +
								      "/queue/rotational");
	rotational = rotational_file.get<bool>();

	const CmdMultipath& cmd_multipath = prober.get_system_info().getCmdMultipath();

	const CmdMultipath::Entry& entry = cmd_multipath.get_entry(get_dm_table_name());

	vendor = entry.vendor;
	model = entry.model;
    }

    void
    Multipath::Impl::probe_pass_1b(Prober& prober)
    {
	const CmdMultipath& cmd_multipath = prober.get_system_info().getCmdMultipath();

	const CmdMultipath::Entry& entry = cmd_multipath.get_entry(get_dm_table_name());

	for (const string& device : entry.devices)
	{
	    BlkDevice* blk_device = BlkDevice::Impl::find_by_any_name(prober.get_system(), device,
								      prober.get_system_info());
	    User::create(prober.get_system(), blk_device, get_non_impl());
	}
    }

    uint64_t
    Multipath::Impl::used_features() const
    {
	return UF_MULTIPATH | Partitionable::Impl::used_features();
    }

    void
    Multipath::Impl::save(xmlNode* node) const
    {
	Partitionable::Impl::save(node);

	setChildValue(node, "vendor", vendor);
	setChildValue(node, "model", model);

	setChildValueIf(node, "rotational", rotational, rotational);
    }

    void
    Multipath::Impl::add_create_actions(Actiongraph::Impl& actiongraph) const
    {
	ST_THROW(Exception("cannot create multipath"));
    }

    void
    Multipath::Impl::add_delete_actions(Actiongraph::Impl& actiongraph) const
    {
	ST_THROW(Exception("cannot delete multipath"));
    }

    bool
    Multipath::Impl::equal(const Device::Impl& rhs_base) const
    {
	const Impl& rhs = dynamic_cast<const Impl&>(rhs_base);

	if (!Partitionable::Impl::equal(rhs))
	    return false;

	return vendor == rhs.vendor && model == rhs.model && rotational == rhs.rotational;
    }

    void
    Multipath::Impl::log_diff(std::ostream& log, const Device::Impl& rhs_base) const
    {
	const Impl& rhs = dynamic_cast<const Impl&>(rhs_base);

	Partitionable::Impl::log_diff(log, rhs);

	storage::log_diff(log, "vendor", vendor, rhs.vendor);
	storage::log_diff(log, "model", model, rhs.model);

	storage::log_diff(log, "rotational", rotational, rhs.rotational);
    }

    void
    Multipath::Impl::print(std::ostream& out) const
    {
	Partitionable::Impl::print(out);

	out << " vendor:" << vendor << " model:" << model;

	if (rotational)
	    out << " rotational";
    }

    void
    Multipath::Impl::process_udev_ids(vector<string>& udev_ids) const
    {
	// See doc/udev.md.

	static const vector<string> allowed_prefixes = { "scsi-", "wwn-", "dm-name-", "dm-uuid-" };

	erase_if(udev_ids, [](const string& udev_id) {
	    return none_of(allowed_prefixes.begin(), allowed_prefixes.end(), [&udev_id](const string& prefix)
			   { return boost::starts_with(udev_id, prefix); });
	});

	stable_partition(udev_ids.begin(), udev_ids.end(), string_starts_with("wwn-"));
	stable_partition(udev_ids.begin(), udev_ids.end(), string_starts_with("scsi-"));
    }

    Text
    Multipath::Impl::do_deactivate_text(Tense tense) const
    {
	Text text = tenser(tense,
			   // TRANSLATORS: displayed before action,
			   // %1$s is replaced by multipath name (e.g. /dev/mapper/36005076305ffc73a00000000000013b4),
			   // %2$s is replaced by size (e.g. 1 TiB)
			   _("Deactivate multipath %1$s (%2$s)"),
			   // TRANSLATORS: displayed during action,
			   // %1$s is replaced by multipath name (e.g. /dev/mapper/36005076305ffc73a00000000000013b4),
			   // %2$s is replaced by size (e.g. 1 TiB)
			   _("Deactivating multipath %1$s (%2$s)"));

	return sformat(text, get_displayname().c_str(), get_size_string().c_str());
    }

    void
    Multipath::Impl::do_deactivate() const
    {
	string cmd_line = MULTIPATHBIN " -f " + quote(get_name());

	SystemCmd cmd(cmd_line, SystemCmd::DoThrow);
    }

}
<|endoftext|>


<commit_before>#include "Album.h"
#include "Database.h"
#include "Debug.h"
#include "WebService.h"

using namespace ost;
using namespace std;

/* constructors/destructor */
WebService::WebService(Database *database) : database(database) {
	root = new XMLNode;
	metadata_search_url = database->loadSetting(METADATA_SEARCH_URL_KEY, METADATA_SEARCH_URL_VALUE, METADATA_SEARCH_URL_DESCRIPTION);
	release_lookup_url = database->loadSetting(RELEASE_LOOKUP_URL_KEY, RELEASE_LOOKUP_URL_VALUE, RELEASE_LOOKUP_URL_DESCRIPTION);
}

WebService::~WebService() {
	clearXMLNode(root);
	delete root;
}

/* methods */
bool WebService::lookupAlbum(Album *album) {
	if (album == NULL || album->mbid.size() != 36 || album->mbid[8] != '-' || album->mbid[13] != '-' || album->mbid[18] != '-' || album->mbid[23] != '-')
		return false;
	string url = release_lookup_url;
	url.append(album->mbid);
	url.append("?type=xml&inc=tracks+puids+artist+release-events+labels+artist-rels+url-rels");
	if (!fetch(url.c_str()))
		return false;
	/* album data */
	if (root->children["metadata"].size() <= 0)
		return false;
	XMLNode *xml_album = root->children["metadata"][0]->children["release"][0];
	album->mbid = xml_album->children["id"][0]->value;
	album->type = xml_album->children["type"][0]->value;
	album->title = xml_album->children["title"][0]->value;
	if (xml_album->children["release-event-list"].size() > 0) {
		album->released = xml_album->children["release-event-list"][0]->children["event"][0]->children["date"][0]->value;
		if (album->released.size() == 10 && album->released[4] == '-' && album->released[7] == '-') {
			/* ok as it is, probably a valid date */
		} else if (album->released.size() == 4) {
			/* only year, make month & day 01 */
			album->released.append("-01-01");
		} else {
			/* possibly not a valid date, ignore it */
			album->released = "";
		}
	}
	/* artist data */
	album->artist.mbid = xml_album->children["artist"][0]->children["id"][0]->value;
	album->artist.name = xml_album->children["artist"][0]->children["name"][0]->value;
	album->artist.sortname = xml_album->children["artist"][0]->children["sort-name"][0]->value;
	/* track data */
	album->tracks.resize(xml_album->children["track-list"][0]->children["track"].size(), Track(album));
	for (vector<XMLNode *>::size_type a = 0; a < xml_album->children["track-list"][0]->children["track"].size(); ++a) {
		/* track data */
		album->tracks[a].mbid = xml_album->children["track-list"][0]->children["track"][a]->children["id"][0]->value;
		album->tracks[a].title = xml_album->children["track-list"][0]->children["track"][a]->children["title"][0]->value;
		album->tracks[a].duration = atoi(xml_album->children["track-list"][0]->children["track"][a]->children["duration"][0]->value.c_str());
		album->tracks[a].tracknumber = a + 1;
		/* track artist data */
		if (xml_album->children["track-list"][0]->children["track"][a]->children["artist"].size() > 0) {
			album->tracks[a].artist.mbid = xml_album->children["track-list"][0]->children["track"][a]->children["artist"][0]->children["id"][0]->value;
			album->tracks[a].artist.name = xml_album->children["track-list"][0]->children["track"][a]->children["artist"][0]->children["name"][0]->value;
			album->tracks[a].artist.sortname = xml_album->children["track-list"][0]->children["track"][a]->children["artist"][0]->children["sort-name"][0]->value;
		} else {
			album->tracks[a].artist.mbid = album->artist.mbid;
			album->tracks[a].artist.name = album->artist.name;
			album->tracks[a].artist.sortname = album->artist.sortname;
		}
	}
	return true;
}

const vector<Metatrack> &WebService::searchMetadata(const string &wsquery) {
	tracks.clear();
	if (wsquery == "")
		return tracks;
	string url = metadata_search_url;
	url.append("?type=xml&");
	url.append(wsquery);
	if (fetch(url.c_str()) && root->children["metadata"].size() > 0 && root->children["metadata"][0]->children["track-list"].size() > 0) {
		for (vector<XMLNode *>::size_type a = 0; a < root->children["metadata"][0]->children["track-list"][0]->children["track"].size(); ++a) {
			if (getMetatrack(root->children["metadata"][0]->children["track-list"][0]->children["track"][a]))
				tracks.push_back(metatrack);
		}
	}
	return tracks;
}

const vector<Metatrack> &WebService::searchPUID(const string &puid) {
	tracks.clear();
	if (puid.size() != 36)
		return tracks;
	string wsquery = "puid=";
	wsquery.append(puid);
	return searchMetadata(wsquery);
}

/* private methods */
void WebService::characters(const unsigned char *text, size_t len) {
	curnode->value = string((char *) text, len);
}

void WebService::clearXMLNode(XMLNode *node) {
	for (map<string, vector<XMLNode *> >::iterator it = node->children.begin(); it != node->children.end(); ++it) {
		for (vector<XMLNode *>::size_type a = 0; a < it->second.size(); ++a) {
			clearXMLNode(it->second[a]);
			delete it->second[a];
		}
	}
}

void WebService::close() {
	URLStream::close();
}

void WebService::endElement(const unsigned char *name) {
	if (curnode != NULL)
		curnode = curnode->parent;
}

bool WebService::fetch(const char *url) {
	char *urle = new char[4096];
	urle = urlEncode(url, urle, 4096);
	Debug::info(urle);
	status = get(urle);
	delete [] urle;
	if (status) {
		//cout << "failed; reason=" << status << endl;
		Debug::warning("Unable to fetch data");
		close();
		return false;
	}
	clearXMLNode(root);
	delete root;
	root = new XMLNode;
	root->parent = NULL;
	root->key = "root";
	root->value = "";
	curnode = root;
	if (!parse())
		Debug::warning("XML is not well formed");
	close();
	//printXML(root, 0);
	return true;
}

bool WebService::getMetatrack(XMLNode *track) {
	metatrack.track_mbid = track->children["id"][0]->value;
	metatrack.track_title = track->children["title"][0]->value;
	if (track->children["duration"].size() > 0)
		metatrack.duration = atoi(track->children["duration"][0]->value.c_str());
	metatrack.artist_mbid = track->children["artist"][0]->children["id"][0]->value;
	metatrack.artist_name = track->children["artist"][0]->children["name"][0]->value;
	metatrack.album_mbid = track->children["release-list"][0]->children["release"][0]->children["id"][0]->value;
	metatrack.album_title = track->children["release-list"][0]->children["release"][0]->children["title"][0]->value;
	string offset = track->children["release-list"][0]->children["release"][0]->children["track-list"][0]->children["offset"][0]->value;
	metatrack.tracknumber = atoi(offset.c_str()) + 1;
	return true;
}

void WebService::printXML(XMLNode *startnode, int indent) const {
	if (startnode == NULL)
		return;
	for (int a = 0; a < indent; ++a)
		cout << "  ";
	if (startnode->parent == NULL)
		cout << startnode->key << ": " << startnode->value << endl;
	else
		cout << startnode->key << " @" << startnode << ": " << startnode->value << " (parent: " << startnode->parent->key << " @" << startnode->parent << ")" << endl;
	for (map<string, vector<XMLNode *> >::iterator it = startnode->children.begin(); it != startnode->children.end(); ++it) {
		for (vector<XMLNode *>::size_type a = 0; a < it->second.size(); ++a)
			printXML(it->second[a], indent + 1);
	}
}

int WebService::read(unsigned char *buffer, size_t len) {
	URLStream::read((char *) buffer, len);
	return gcount();
}

void WebService::startElement(const unsigned char *name, const unsigned char **attr) {
	XMLNode *childnode = new XMLNode;
	childnode->parent = curnode;
	childnode->key = (char *) name;
	childnode->value = "";
	curnode->children[childnode->key].push_back(childnode);
	curnode = curnode->children[childnode->key][curnode->children[childnode->key].size() - 1];
	if (attr != NULL) {
		while (*attr != NULL) {
			childnode = new XMLNode;
			childnode->parent = curnode;
			childnode->key = (char *) *(attr++);
			childnode->value = (char *) *(attr++);
			curnode->children[childnode->key].push_back(childnode);
		}
	}
}
<commit_msg>actually checking the xml before fetching data, so it won't die horribly when some xml element is missing<commit_after>#include "Album.h"
#include "Database.h"
#include "Debug.h"
#include "WebService.h"

using namespace ost;
using namespace std;

/* constructors/destructor */
WebService::WebService(Database *database) : database(database) {
	root = new XMLNode;
	metadata_search_url = database->loadSetting(METADATA_SEARCH_URL_KEY, METADATA_SEARCH_URL_VALUE, METADATA_SEARCH_URL_DESCRIPTION);
	release_lookup_url = database->loadSetting(RELEASE_LOOKUP_URL_KEY, RELEASE_LOOKUP_URL_VALUE, RELEASE_LOOKUP_URL_DESCRIPTION);
}

WebService::~WebService() {
	clearXMLNode(root);
	delete root;
}

/* methods */
bool WebService::lookupAlbum(Album *album) {
	if (album == NULL || album->mbid.size() != 36 || album->mbid[8] != '-' || album->mbid[13] != '-' || album->mbid[18] != '-' || album->mbid[23] != '-')
		return false;
	string url = release_lookup_url;
	url.append(album->mbid);
	url.append("?type=xml&inc=tracks+puids+artist+release-events+labels+artist-rels+url-rels");
	if (!fetch(url.c_str()))
		return false;
	/* album data */
	if (root->children["metadata"].size() <= 0 || root->children["metadata"][0]->children["release"].size() <= 0)
		return false;
	XMLNode *tmp = root->children["metadata"][0]->children["release"][0];
	album->mbid = (tmp->children["id"].size() > 0) ? tmp->children["id"][0]->value : "";
	album->type = (tmp->children["type"].size() > 0) ? tmp->children["type"][0]->value : "";
	album->title = (tmp->children["title"].size() > 0) ? tmp->children["title"][0]->value : "";
	if (tmp->children["release-event-list"].size() > 0 && tmp->children["release-event-list"][0]->children["event"].size() > 0 && tmp->children["release-event-list"][0]->children["event"][0]->children["date"].size() > 0) {
		album->released = tmp->children["release-event-list"][0]->children["event"][0]->children["date"][0]->value;
		if (album->released.size() == 10 && album->released[4] == '-' && album->released[7] == '-') {
			/* ok as it is, probably a valid date */
		} else if (album->released.size() == 4) {
			/* only year, make month & day 01 */
			album->released.append("-01-01");
		} else {
			/* possibly not a valid date, ignore it */
			album->released = "";
		}
	}
	/* artist data */
	if (tmp->children["artist"].size() <= 0)
		return false;
	XMLNode *tmp2 = tmp->children["artist"][0];
	album->artist.mbid = (tmp2->children["id"].size() > 0) ? tmp2->children["id"][0]->value : "";
	album->artist.name = (tmp2->children["name"].size() > 0) ? tmp2->children["name"][0]->value : "";
	album->artist.sortname = (tmp2->children["sort-name"].size() > 0) ? tmp2->children["sort-name"][0]->value : "";
	/* track data */
	if (tmp->children["track-list"].size() <= 0 || tmp->children["track-list"][0]->children["track"].size() <= 0)
		return false;
	tmp = tmp->children["track-list"][0];
	album->tracks.resize(tmp->children["track"].size(), Track(album));
	for (vector<XMLNode *>::size_type a = 0; a < tmp->children["track"].size(); ++a) {
		/* track data */
		tmp2 = tmp->children["track"][a];
		album->tracks[a].mbid = (tmp2->children["id"].size() > 0) ? tmp2->children["id"][0]->value : "";
		album->tracks[a].title = (tmp2->children["title"].size() > 0) ? tmp2->children["title"][0]->value : "";
		album->tracks[a].duration = (tmp2->children["duration"].size() > 0) ? atoi(tmp2->children["duration"][0]->value.c_str()) : 0;
		album->tracks[a].tracknumber = a + 1;
		/* track artist data */
		if (tmp2->children["artist"].size() > 0) {
			tmp2 = tmp2->children["artist"][0];
			album->tracks[a].artist.mbid = (tmp2->children["id"].size() > 0) ? tmp2->children["id"][0]->value : "";
			album->tracks[a].artist.name = (tmp2->children["name"].size() > 0) ? tmp2->children["name"][0]->value : "";
			album->tracks[a].artist.sortname = (tmp2->children["sort-name"].size() > 0) ? tmp2->children["sort-name"][0]->value : "";
		} else {
			album->tracks[a].artist.mbid = album->artist.mbid;
			album->tracks[a].artist.name = album->artist.name;
			album->tracks[a].artist.sortname = album->artist.sortname;
		}
	}
	return true;
}

const vector<Metatrack> &WebService::searchMetadata(const string &wsquery) {
	tracks.clear();
	if (wsquery == "")
		return tracks;
	string url = metadata_search_url;
	url.append("?type=xml&");
	url.append(wsquery);
	if (fetch(url.c_str()) && root->children["metadata"].size() > 0 && root->children["metadata"][0]->children["track-list"].size() > 0) {
		for (vector<XMLNode *>::size_type a = 0; a < root->children["metadata"][0]->children["track-list"][0]->children["track"].size(); ++a) {
			if (getMetatrack(root->children["metadata"][0]->children["track-list"][0]->children["track"][a]))
				tracks.push_back(metatrack);
		}
	}
	return tracks;
}

const vector<Metatrack> &WebService::searchPUID(const string &puid) {
	tracks.clear();
	if (puid.size() != 36)
		return tracks;
	string wsquery = "puid=";
	wsquery.append(puid);
	return searchMetadata(wsquery);
}

/* private methods */
void WebService::characters(const unsigned char *text, size_t len) {
	curnode->value = string((char *) text, len);
}

void WebService::clearXMLNode(XMLNode *node) {
	for (map<string, vector<XMLNode *> >::iterator it = node->children.begin(); it != node->children.end(); ++it) {
		for (vector<XMLNode *>::size_type a = 0; a < it->second.size(); ++a) {
			clearXMLNode(it->second[a]);
			delete it->second[a];
		}
	}
}

void WebService::close() {
	URLStream::close();
}

void WebService::endElement(const unsigned char *name) {
	if (curnode != NULL)
		curnode = curnode->parent;
}

bool WebService::fetch(const char *url) {
	char *urle = new char[4096];
	urle = urlEncode(url, urle, 4096);
	Debug::info(urle);
	status = get(urle);
	delete [] urle;
	if (status) {
		//cout << "failed; reason=" << status << endl;
		Debug::warning("Unable to fetch data");
		close();
		return false;
	}
	clearXMLNode(root);
	delete root;
	root = new XMLNode;
	root->parent = NULL;
	root->key = "root";
	root->value = "";
	curnode = root;
	if (!parse())
		Debug::warning("XML is not well formed");
	close();
	//printXML(root, 0);
	return true;
}

bool WebService::getMetatrack(XMLNode *track) {
	if (track == NULL)
		return false;
	metatrack.track_mbid = (track->children["id"].size() > 0) ? track->children["id"][0]->value : "";
	metatrack.track_title = (track->children["title"].size() > 0) ? track->children["title"][0]->value : "";
	metatrack.duration = (track->children["duration"].size() > 0) ? atoi(track->children["duration"][0]->value.c_str()) : 0;
	if (track->children["artist"].size() <= 0) {
		metatrack.artist_mbid = "";
		metatrack.artist_name = "";
		metatrack.album_mbid = "";
		metatrack.album_title = "";
		metatrack.tracknumber = 0;
		return false;
	}
	XMLNode *tmp = track->children["artist"][0];
	metatrack.artist_mbid = (tmp->children["id"].size() > 0) ? tmp->children["id"][0]->value : "";
	metatrack.artist_name = (tmp->children["name"].size() > 0) ? tmp->children["name"][0]->value : "";
	if (track->children["release-list"].size() <= 0 || track->children["release-list"][0]->children["release"].size() <= 0) {
		metatrack.album_mbid = "";
		metatrack.album_title = "";
		metatrack.tracknumber = 0;
		return false;
	}
	tmp = track->children["release-list"][0]->children["release"][0];
	metatrack.album_mbid = (tmp->children["id"].size() > 0) ? tmp->children["id"][0]->value : "";
	metatrack.album_title = (tmp->children["title"].size() > 0) ? tmp->children["title"][0]->value : "";
	metatrack.tracknumber = (tmp->children["track-list"].size() > 0 && tmp->children["track-list"][0]->children["offset"].size() > 0) ? atoi(tmp->children["track-list"][0]->children["offset"][0]->value.c_str()) + 1: 0;
	return true;
}

void WebService::printXML(XMLNode *startnode, int indent) const {
	if (startnode == NULL)
		return;
	for (int a = 0; a < indent; ++a)
		cout << "  ";
	if (startnode->parent == NULL)
		cout << startnode->key << ": " << startnode->value << endl;
	else
		cout << startnode->key << " @" << startnode << ": " << startnode->value << " (parent: " << startnode->parent->key << " @" << startnode->parent << ")" << endl;
	for (map<string, vector<XMLNode *> >::iterator it = startnode->children.begin(); it != startnode->children.end(); ++it) {
		for (vector<XMLNode *>::size_type a = 0; a < it->second.size(); ++a)
			printXML(it->second[a], indent + 1);
	}
}

int WebService::read(unsigned char *buffer, size_t len) {
	URLStream::read((char *) buffer, len);
	return gcount();
}

void WebService::startElement(const unsigned char *name, const unsigned char **attr) {
	XMLNode *childnode = new XMLNode;
	childnode->parent = curnode;
	childnode->key = (char *) name;
	childnode->value = "";
	curnode->children[childnode->key].push_back(childnode);
	curnode = curnode->children[childnode->key][curnode->children[childnode->key].size() - 1];
	if (attr != NULL) {
		while (*attr != NULL) {
			childnode = new XMLNode;
			childnode->parent = curnode;
			childnode->key = (char *) *(attr++);
			childnode->value = (char *) *(attr++);
			curnode->children[childnode->key].push_back(childnode);
		}
	}
}
<|endoftext|>