<commit_before>/*************************************************************************
 *
 *  $RCSfile: c_rcode.cxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: vg $ $Date: 2003-12-17 18:35:42 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#include <precomp.h>
#include "c_rcode.hxx"

// NOT FULLY DECLARED SERVICES
#include <ary/cpp/c_rwgate.hxx>
#include <ary/cpp/c_namesp.hxx>
// #include <ary/cpp/c_groups.hxx>
#include <ary/loc/l_rwgate.hxx>
#include "cpp_pe.hxx"
#include <adc_cl.hxx>
#include <x_parse.hxx>
#include "pe_file.hxx"

#ifdef IRIX
// see i19418 incomplete type for cpp::PE_Namespace
#include "pe_namsp.hxx"
#include "pe_ignor.hxx"
#include "pe_defs.hxx"
#include "pe_vafu.hxx"
#include "pe_tydef.hxx"
#include "pe_tpltp.hxx"
#include "pe_type.hxx"
#include "pe_vari.hxx"
#include "pe_funct.hxx"
#include "pe_enum.hxx"
#include "pe_param.hxx"
#include "pe_class.hxx"
#include "pe_expression.hxx"
#include "pe_base.hxx"
#include "pe_enum.hxx"
#include "pe_enval.hxx"
#endif

const uintt C_nNO_TRY = uintt(-1);

namespace cpp {

CodeExplorer::CodeExplorer( ary::cpp::RwGate & io_rAryGate )
    :   aGlobalParseContext(io_rAryGate),
        // aEnvironments,
        pPE_File(0),
        pGate(&io_rAryGate),
        dpCurToken(0)
{
    pPE_File = new PE_File( aGlobalParseContext );
}

CodeExplorer::~CodeExplorer()
{
}

void
CodeExplorer::StartNewFile()
{
    csv::erase_container(aEnvironments);

    aEnvironments.push_back( pPE_File.MutablePtr() );
    pPE_File->Enter(push);
}

void
CodeExplorer::Process_Token( DYN cpp::Token & let_drToken )
{
if (DEBUG_ShowTokens())
{
    Cout() << let_drToken.Text() << Endl();
}
    dpCurToken = &let_drToken;
    aGlobalParseContext.ResetResult();

    do {
        CurToken().Trigger( CurEnv() );
        AcknowledgeResult();
    } while ( dpCurToken );
}

void
CodeExplorer::AcknowledgeResult()
{
    if (CurResult().eDone == done)
        dpCurToken = 0;

    switch ( CurResult().eStackAction )
    {
        case stay:
                break;
        case push:
                CurEnv().Leave(push);
                aEnvironments.push_back( &PushEnv() );
                PushEnv().Enter(push);
                break;
        case pop_success:
                CurEnv().Leave(pop_success);
                aEnvironments.pop_back();
                CurEnv().Enter(pop_success);
                break;
        case pop_failure:
        {
                Cpp_PE * pRecover = 0;
                do {
                    CurEnv().Leave(pop_failure);
                    aEnvironments.pop_back();
                    if ( aEnvironments.empty() )
                        break;
                    pRecover = CurEnv().Handle_ChildFailure();
                } while ( pRecover == 0 );
                if ( pRecover != 0 )
                {
                    aEnvironments.push_back(pRecover);
                    pRecover->Enter(push);
                }
                else
                {
                    throw X_Parser( X_Parser::x_UnexpectedToken, CurToken().Text(), aGlobalParseContext.CurFileName(), aGlobalParseContext.LineCount() );
                }
        }       break;
        default:
            csv_assert(false);
    }   // end switch(CurResult().eStackAction)
}

ary::Cid
CodeExplorer::CurFile() const
{
    // KORR
    return 0;
}

const Token &
CodeExplorer::CurToken() const
{
    csv_assert(dpCurToken);

    return *dpCurToken;
}

Cpp_PE &
CodeExplorer::CurEnv() const
{
    csv_assert(aEnvironments.size() > 0);
    csv_assert(aEnvironments.back() != 0);

    return *aEnvironments.back();
}

Cpp_PE &
CodeExplorer::PushEnv() const
{
    TokenProcessing_Result & rCurResult = const_cast< TokenProcessing_Result& >(aGlobalParseContext.CurResult());
    Cpp_PE * ret = dynamic_cast< Cpp_PE* >(rCurResult.pEnv2Push);
    csv_assert( ret != 0 );
    return *ret;
}

}   // namespace cpp

<commit_msg>INTEGRATION: CWS ooo19126 (1.2.50); FILE MERGED 2005/09/05 13:11:37 rt 1.2.50.1: #i54170# Change license header: remove SISSL<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: c_rcode.cxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-07 18:15:13 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#include <precomp.h>
#include "c_rcode.hxx"

// NOT FULLY DECLARED SERVICES
#include <ary/cpp/c_rwgate.hxx>
#include <ary/cpp/c_namesp.hxx>
// #include <ary/cpp/c_groups.hxx>
#include <ary/loc/l_rwgate.hxx>
#include "cpp_pe.hxx"
#include <adc_cl.hxx>
#include <x_parse.hxx>
#include "pe_file.hxx"

#ifdef IRIX
// see i19418 incomplete type for cpp::PE_Namespace
#include "pe_namsp.hxx"
#include "pe_ignor.hxx"
#include "pe_defs.hxx"
#include "pe_vafu.hxx"
#include "pe_tydef.hxx"
#include "pe_tpltp.hxx"
#include "pe_type.hxx"
#include "pe_vari.hxx"
#include "pe_funct.hxx"
#include "pe_enum.hxx"
#include "pe_param.hxx"
#include "pe_class.hxx"
#include "pe_expression.hxx"
#include "pe_base.hxx"
#include "pe_enum.hxx"
#include "pe_enval.hxx"
#endif

const uintt C_nNO_TRY = uintt(-1);

namespace cpp {

CodeExplorer::CodeExplorer( ary::cpp::RwGate & io_rAryGate )
    :   aGlobalParseContext(io_rAryGate),
        // aEnvironments,
        pPE_File(0),
        pGate(&io_rAryGate),
        dpCurToken(0)
{
    pPE_File = new PE_File( aGlobalParseContext );
}

CodeExplorer::~CodeExplorer()
{
}

void
CodeExplorer::StartNewFile()
{
    csv::erase_container(aEnvironments);

    aEnvironments.push_back( pPE_File.MutablePtr() );
    pPE_File->Enter(push);
}

void
CodeExplorer::Process_Token( DYN cpp::Token & let_drToken )
{
if (DEBUG_ShowTokens())
{
    Cout() << let_drToken.Text() << Endl();
}
    dpCurToken = &let_drToken;
    aGlobalParseContext.ResetResult();

    do {
        CurToken().Trigger( CurEnv() );
        AcknowledgeResult();
    } while ( dpCurToken );
}

void
CodeExplorer::AcknowledgeResult()
{
    if (CurResult().eDone == done)
        dpCurToken = 0;

    switch ( CurResult().eStackAction )
    {
        case stay:
                break;
        case push:
                CurEnv().Leave(push);
                aEnvironments.push_back( &PushEnv() );
                PushEnv().Enter(push);
                break;
        case pop_success:
                CurEnv().Leave(pop_success);
                aEnvironments.pop_back();
                CurEnv().Enter(pop_success);
                break;
        case pop_failure:
        {
                Cpp_PE * pRecover = 0;
                do {
                    CurEnv().Leave(pop_failure);
                    aEnvironments.pop_back();
                    if ( aEnvironments.empty() )
                        break;
                    pRecover = CurEnv().Handle_ChildFailure();
                } while ( pRecover == 0 );
                if ( pRecover != 0 )
                {
                    aEnvironments.push_back(pRecover);
                    pRecover->Enter(push);
                }
                else
                {
                    throw X_Parser( X_Parser::x_UnexpectedToken, CurToken().Text(), aGlobalParseContext.CurFileName(), aGlobalParseContext.LineCount() );
                }
        }       break;
        default:
            csv_assert(false);
    }   // end switch(CurResult().eStackAction)
}

ary::Cid
CodeExplorer::CurFile() const
{
    // KORR
    return 0;
}

const Token &
CodeExplorer::CurToken() const
{
    csv_assert(dpCurToken);

    return *dpCurToken;
}

Cpp_PE &
CodeExplorer::CurEnv() const
{
    csv_assert(aEnvironments.size() > 0);
    csv_assert(aEnvironments.back() != 0);

    return *aEnvironments.back();
}

Cpp_PE &
CodeExplorer::PushEnv() const
{
    TokenProcessing_Result & rCurResult = const_cast< TokenProcessing_Result& >(aGlobalParseContext.CurResult());
    Cpp_PE * ret = dynamic_cast< Cpp_PE* >(rCurResult.pEnv2Push);
    csv_assert( ret != 0 );
    return *ret;
}

}   // namespace cpp

<|endoftext|>


<commit_before>/**
 * @file Cosa/IOStream.cpp
 * @version 1.0
 *
 * @section License
 * Copyright (C) 2012-2014, Mikael Patel
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * This file is part of the Arduino Che Cosa project.
 */

#include "Cosa/IOStream.hh"
#include "Cosa/Power.hh"
#include <ctype.h>

IOStream::IOStream(Device* dev) :
  m_dev(dev),
  m_base(dec),
  m_width(6),
  m_prec(4)
{}

IOStream::IOStream() :
  m_dev(NULL),
  m_base(dec),
  m_width(6),
  m_prec(4)
{}

IOStream::Device*
IOStream::set_device(Device* dev)
{
  if (dev == NULL) return (m_dev);
  Device* previous = m_dev;
  m_dev = dev;
  return (previous);
}

void
IOStream::print(int n, Base base)
{
  if (base != bcd) {
    if (base != dec) print_prefix(base);
    char buf[sizeof(int) * CHARBITS + 1];
    print(itoa(n, buf, base));
  }
  else {
    print((char) ('0' + ((n >> 4) & 0xf)));
    print((char) ('0' + (n & 0xf)));
  }
}

void
IOStream::print(long int n, Base base)
{
  if (base != dec) print_prefix(base);
  char buf[sizeof(long int) * CHARBITS + 1];
  print(ltoa(n, buf, base));
}

void
IOStream::print(unsigned int n, Base base)
{
  if (base != dec) print_prefix(base);
  char buf[sizeof(int) * CHARBITS + 1];
  print(utoa(n, buf, base));
}

void
IOStream::print(unsigned long int n, Base base)
{
  if (base != dec) print_prefix(base);
  char buf[sizeof(long int) * CHARBITS + 1];
  print(ultoa(n, buf, base));
}

void
IOStream::print(unsigned int n, uint8_t digits, Base base)
{
  char buf[sizeof(int) * CHARBITS + 1];
  utoa(n, buf, base);
  for (uint8_t length = strlen(buf); length < digits; length++)
    print('0');
  print(buf);
}

void
IOStream::print(unsigned long int n, uint8_t digits, Base base)
{
  char buf[sizeof(long int) * CHARBITS + 1];
  ultoa(n, buf, base);
  for (uint8_t length = strlen(buf); length < digits; length++)
    print('0');
  print(buf);
}

void
IOStream::print(double value, int8_t width, uint8_t prec)
{
  const uint8_t BUF_MAX = 33;
  char buf[BUF_MAX];
  dtostrf(value, width, prec, buf);
  print(buf);
}

void
IOStream::print(IOStream::Device* buffer)
{
  int c;
  while ((c = buffer->getchar()) != EOF)
    print((char) c);
}

void
IOStream::print_prefix(Base base)
{
  if (base == hex)
    print_P(PSTR("0x"));
  else if (base == bin)
    print_P(PSTR("0b"));
  else if (base == oct)
    print_P(PSTR("0"));
}

void
IOStream::print(uint32_t src, const void *ptr, size_t size, Base base, uint8_t max)
{
  Base b = (base != dec ? hex : dec);
  uint8_t w = (base == hex ? 2 : (base == bin ? 8 : 3));
  uint8_t* p = (uint8_t*) ptr;
  uint8_t n = 0;
  print(src, 6, b);
  print_P(PSTR(": "));
  while (size--) {
    print((unsigned int) *p++, w, base);
    src += 1;
    if (++n < max) {
      print_P(PSTR(" "));
    }
    else {
      println();
      n = 0;
      if (size > 0) {
	print(src, 6, b);
	print_P(PSTR(": "));
      }
    }
  }
  if (n != 0) println();
}

void
IOStream::vprintf_P(const char* format, va_list args)
{
  const char* s = format;
  uint8_t is_signed;
  Base base;
  char c;
  while ((c = pgm_read_byte(s++)) != 0) {
    if (c == '%') {
      is_signed = 1;
      base = dec;
    next:
      c = pgm_read_byte(s++);
      if (c == 0) s--;
      switch (c) {
      case 'b':
	base = bin;
	goto next;
      case 'B':
	base = bcd;
	goto next;
      case 'o':
	base = oct;
	goto next;
      case 'h':
      case 'x':
	base = hex;
	goto next;
      case 'u':
	is_signed = 0;
	goto next;
      case 'c':
	print((char) va_arg(args, int));
	continue;
      case 'p':
	print(va_arg(args, void*));
	continue;
      case 's':
	print(va_arg(args, char*));
	continue;
      case 'S':
	print_P(va_arg(args, const char*));
	continue;
      case 'd':
	if (is_signed)
	  print(va_arg(args, int), base);
	else
	  print(va_arg(args, unsigned int), base);
	continue;
      case 'l':
	if (is_signed)
	  print(va_arg(args, long int), base);
	else
	  print(va_arg(args, unsigned long int), base);
	continue;
      };
    }
    print(c);
  }
}

char*
IOStream::scan(char *s, size_t count)
{
  if (m_dev == NULL) return (NULL);
  char* res = s;

  // Skip whitespace
  int c = m_dev->peekchar();
  while (c <= ' ' && c != '\n') {
    if (c == EOF) return (NULL);
    c = m_dev->getchar();
    c = m_dev->peekchar();
  }
  c = m_dev->getchar();

  // Scan the token; identifier, number or special character
  count -= 1;
  *s++ = c;
  if (isalpha(c)) {
    while (count--) {
      c = m_dev->peekchar();
      if (!isalnum(c)) break;
      c = m_dev->getchar();
      *s++ = c;
    }
  }
  else if (isdigit(c) || c == '-') {
    while (count--) {
      c = m_dev->peekchar();
      if (!isdigit(c)) break;
      c = m_dev->getchar();
      *s++ = c;
    }
  }
  *s = 0;
  return (res);
}

char*
IOStream::readline(char* buf, size_t size)
{
  if (m_dev == NULL) return (NULL);
  int count = m_dev->available();
  if (count <= 0) return (NULL);
  size_t len = strlen(buf);
  char* s = buf + len;
  char c = 0;
  while (count--) {
    c = m_dev->getchar();
    if (c == '\b' || c == 127) {
      if (len > 0) {
	print_P(PSTR("\b \b"));
	len -= 1;
	s -= 1;
      }
    }
    else {
      print(c);
      if (c == '\r') {
	if (m_dev->get_eol() == CRLF_MODE) continue;
	c = '\n';
	print(c);
      }
      if (len < size) {
	len += 1;
	*s++ = c;
      }
      if (c == '\n') break;
    }
  }
  *s = 0;
  return (c == '\n' ? buf : NULL);
}

// IOStream::Device IOStream::Device::null;

int
IOStream::Device::available()
{
  return (0);
}

int
IOStream::Device::room()
{
  return (0);
}

int
IOStream::Device::putchar(char c)
{
  int res = write(&c, sizeof(c));
  return (res == sizeof(c) ? c & 0xff : res);
}

int
IOStream::Device::puts(const char* s)
{
  return (write(s, strlen(s)));
}

int
IOStream::Device::puts_P(const char* s)
{
  return (write_P(s, strlen_P(s)));
}

int
IOStream::Device::write(const void* buf, size_t size)
{
  char* bp = (char*) buf;
  size_t n = 0;
  for(; n < size; n++)
    if (putchar(*bp++) < 0)
      break;
  return (n);
}

int
IOStream::Device::write_P(const void* buf, size_t size)
{
  char* bp = (char*) buf;
  size_t n = 0;
  for(; n < size; n++)
    if (putchar(pgm_read_byte(bp++)) < 0)
      break;
  return (n);
}

int
IOStream::Device::write(const iovec_t* vec)
{
  size_t size = 0;
  for (const iovec_t* vp = vec; vp->buf != NULL; vp++) {
    size_t res = (size_t) write(vp->buf, vp->size);
    if (res == 0) break;
    size += res;
  }
  return (size);
}

int
IOStream::Device::peekchar()
{
  return (EOF);
}

int
IOStream::Device::peekchar(char c)
{
  UNUSED(c);
  return (EOF);
}

int
IOStream::Device::getchar()
{
  return (EOF);
}

char*
IOStream::Device::gets(char *s, size_t count)
{
  char* res = s;
  while (--count) {
    int c = getchar();
    if (c == EOF) {
      if (m_blocking) {
	while (c == EOF) {
	  yield();
	  c = getchar();
	}
      }
      else break;
    }
    if (c == '\r') {
      if (m_eol == CRLF_MODE) continue;
      c = '\n';
    }
    *s++ = c;
    if (c == '\n') break;
  }
  *s = 0;
  return (s == res ? NULL : res);
}

int
IOStream::Device::read(void* buf, size_t size)
{
  char* bp = (char*) buf;
  size_t n = 0;
  for (; n < size; n++) {
    int c = getchar();
    if (c < 0) break;
    *bp++ = c;
  }
  return (n);
}

int
IOStream::Device::read(iovec_t* vec)
{
  size_t size = 0;
  for (const iovec_t* vp = vec; vp->buf != NULL; vp++) {
    size_t res = (size_t) read(vp->buf, vp->size);
    if (res == 0) break;
    size += res;
  }
  return (size);
}

int
IOStream::Device::flush()
{
  return (EOF);
}
<commit_msg>Remove ESC to make function key codes visible.<commit_after>/**
 * @file Cosa/IOStream.cpp
 * @version 1.0
 *
 * @section License
 * Copyright (C) 2012-2014, Mikael Patel
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * This file is part of the Arduino Che Cosa project.
 */

#include "Cosa/IOStream.hh"
#include "Cosa/Power.hh"
#include <ctype.h>

IOStream::IOStream(Device* dev) :
  m_dev(dev),
  m_base(dec),
  m_width(6),
  m_prec(4)
{}

IOStream::IOStream() :
  m_dev(NULL),
  m_base(dec),
  m_width(6),
  m_prec(4)
{}

IOStream::Device*
IOStream::set_device(Device* dev)
{
  if (dev == NULL) return (m_dev);
  Device* previous = m_dev;
  m_dev = dev;
  return (previous);
}

void
IOStream::print(int n, Base base)
{
  if (base != bcd) {
    if (base != dec) print_prefix(base);
    char buf[sizeof(int) * CHARBITS + 1];
    print(itoa(n, buf, base));
  }
  else {
    print((char) ('0' + ((n >> 4) & 0xf)));
    print((char) ('0' + (n & 0xf)));
  }
}

void
IOStream::print(long int n, Base base)
{
  if (base != dec) print_prefix(base);
  char buf[sizeof(long int) * CHARBITS + 1];
  print(ltoa(n, buf, base));
}

void
IOStream::print(unsigned int n, Base base)
{
  if (base != dec) print_prefix(base);
  char buf[sizeof(int) * CHARBITS + 1];
  print(utoa(n, buf, base));
}

void
IOStream::print(unsigned long int n, Base base)
{
  if (base != dec) print_prefix(base);
  char buf[sizeof(long int) * CHARBITS + 1];
  print(ultoa(n, buf, base));
}

void
IOStream::print(unsigned int n, uint8_t digits, Base base)
{
  char buf[sizeof(int) * CHARBITS + 1];
  utoa(n, buf, base);
  for (uint8_t length = strlen(buf); length < digits; length++)
    print('0');
  print(buf);
}

void
IOStream::print(unsigned long int n, uint8_t digits, Base base)
{
  char buf[sizeof(long int) * CHARBITS + 1];
  ultoa(n, buf, base);
  for (uint8_t length = strlen(buf); length < digits; length++)
    print('0');
  print(buf);
}

void
IOStream::print(double value, int8_t width, uint8_t prec)
{
  const uint8_t BUF_MAX = 33;
  char buf[BUF_MAX];
  dtostrf(value, width, prec, buf);
  print(buf);
}

void
IOStream::print(IOStream::Device* buffer)
{
  int c;
  while ((c = buffer->getchar()) != EOF)
    print((char) c);
}

void
IOStream::print_prefix(Base base)
{
  if (base == hex)
    print_P(PSTR("0x"));
  else if (base == bin)
    print_P(PSTR("0b"));
  else if (base == oct)
    print_P(PSTR("0"));
}

void
IOStream::print(uint32_t src, const void *ptr, size_t size, Base base, uint8_t max)
{
  Base b = (base != dec ? hex : dec);
  uint8_t w = (base == hex ? 2 : (base == bin ? 8 : 3));
  uint8_t* p = (uint8_t*) ptr;
  uint8_t n = 0;
  print(src, 6, b);
  print_P(PSTR(": "));
  while (size--) {
    print((unsigned int) *p++, w, base);
    src += 1;
    if (++n < max) {
      print_P(PSTR(" "));
    }
    else {
      println();
      n = 0;
      if (size > 0) {
	print(src, 6, b);
	print_P(PSTR(": "));
      }
    }
  }
  if (n != 0) println();
}

void
IOStream::vprintf_P(const char* format, va_list args)
{
  const char* s = format;
  uint8_t is_signed;
  Base base;
  char c;
  while ((c = pgm_read_byte(s++)) != 0) {
    if (c == '%') {
      is_signed = 1;
      base = dec;
    next:
      c = pgm_read_byte(s++);
      if (c == 0) s--;
      switch (c) {
      case 'b':
	base = bin;
	goto next;
      case 'B':
	base = bcd;
	goto next;
      case 'o':
	base = oct;
	goto next;
      case 'h':
      case 'x':
	base = hex;
	goto next;
      case 'u':
	is_signed = 0;
	goto next;
      case 'c':
	print((char) va_arg(args, int));
	continue;
      case 'p':
	print(va_arg(args, void*));
	continue;
      case 's':
	print(va_arg(args, char*));
	continue;
      case 'S':
	print_P(va_arg(args, const char*));
	continue;
      case 'd':
	if (is_signed)
	  print(va_arg(args, int), base);
	else
	  print(va_arg(args, unsigned int), base);
	continue;
      case 'l':
	if (is_signed)
	  print(va_arg(args, long int), base);
	else
	  print(va_arg(args, unsigned long int), base);
	continue;
      };
    }
    print(c);
  }
}

char*
IOStream::scan(char *s, size_t count)
{
  if (m_dev == NULL) return (NULL);
  char* res = s;

  // Skip whitespace
  int c = m_dev->peekchar();
  while (c <= ' ' && c != '\n') {
    if (c == EOF) return (NULL);
    c = m_dev->getchar();
    c = m_dev->peekchar();
  }
  c = m_dev->getchar();

  // Scan the token; identifier, number or special character
  count -= 1;
  *s++ = c;
  if (isalpha(c)) {
    while (count--) {
      c = m_dev->peekchar();
      if (!isalnum(c)) break;
      c = m_dev->getchar();
      *s++ = c;
    }
  }
  else if (isdigit(c) || c == '-') {
    while (count--) {
      c = m_dev->peekchar();
      if (!isdigit(c)) break;
      c = m_dev->getchar();
      *s++ = c;
    }
  }
  *s = 0;
  return (res);
}

char*
IOStream::readline(char* buf, size_t size)
{
  const char DEL = 127;
  const char ESC = 27;
  if (m_dev == NULL) return (NULL);
  int count = m_dev->available();
  if (count <= 0) return (NULL);
  size_t len = strlen(buf);
  char* s = buf + len;
  char c = 0;
  while (count--) {
    c = m_dev->getchar();
    if (c == ESC) continue;
    if (c == '\b' || c == DEL) {
      if (len > 0) {
	print_P(PSTR("\b \b"));
	len -= 1;
	s -= 1;
      }
    }
    else {
      if (c == '\r') {
	print(c);
	if (m_dev->get_eol() == CRLF_MODE) continue;
	c = '\n';
      }
      print(c);
      if (len < size) {
	len += 1;
	*s++ = c;
      }
      if (c == '\n') break;
    }
  }
  *s = 0;
  return (c == '\n' ? buf : NULL);
}

// IOStream::Device IOStream::Device::null;

int
IOStream::Device::available()
{
  return (0);
}

int
IOStream::Device::room()
{
  return (0);
}

int
IOStream::Device::putchar(char c)
{
  int res = write(&c, sizeof(c));
  return (res == sizeof(c) ? c & 0xff : res);
}

int
IOStream::Device::puts(const char* s)
{
  return (write(s, strlen(s)));
}

int
IOStream::Device::puts_P(const char* s)
{
  return (write_P(s, strlen_P(s)));
}

int
IOStream::Device::write(const void* buf, size_t size)
{
  char* bp = (char*) buf;
  size_t n = 0;
  for(; n < size; n++)
    if (putchar(*bp++) < 0)
      break;
  return (n);
}

int
IOStream::Device::write_P(const void* buf, size_t size)
{
  char* bp = (char*) buf;
  size_t n = 0;
  for(; n < size; n++)
    if (putchar(pgm_read_byte(bp++)) < 0)
      break;
  return (n);
}

int
IOStream::Device::write(const iovec_t* vec)
{
  size_t size = 0;
  for (const iovec_t* vp = vec; vp->buf != NULL; vp++) {
    size_t res = (size_t) write(vp->buf, vp->size);
    if (res == 0) break;
    size += res;
  }
  return (size);
}

int
IOStream::Device::peekchar()
{
  return (EOF);
}

int
IOStream::Device::peekchar(char c)
{
  UNUSED(c);
  return (EOF);
}

int
IOStream::Device::getchar()
{
  return (EOF);
}

char*
IOStream::Device::gets(char *s, size_t count)
{
  char* res = s;
  while (--count) {
    int c = getchar();
    if (c == EOF) {
      if (m_blocking) {
	while (c == EOF) {
	  yield();
	  c = getchar();
	}
      }
      else break;
    }
    if (c == '\r') {
      if (m_eol == CRLF_MODE) continue;
      c = '\n';
    }
    *s++ = c;
    if (c == '\n') break;
  }
  *s = 0;
  return (s == res ? NULL : res);
}

int
IOStream::Device::read(void* buf, size_t size)
{
  char* bp = (char*) buf;
  size_t n = 0;
  for (; n < size; n++) {
    int c = getchar();
    if (c < 0) break;
    *bp++ = c;
  }
  return (n);
}

int
IOStream::Device::read(iovec_t* vec)
{
  size_t size = 0;
  for (const iovec_t* vp = vec; vp->buf != NULL; vp++) {
    size_t res = (size_t) read(vp->buf, vp->size);
    if (res == 0) break;
    size += res;
  }
  return (size);
}

int
IOStream::Device::flush()
{
  return (EOF);
}
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  $RCSfile: pe_base.hxx,v $
 *
 *  $Revision: 1.1.1.1 $
 *
 *  last change: $Author: np $ $Date: 2002-03-08 14:45:30 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#ifndef ADC_CPP_PE_BASE_HXX
#define ADC_CPP_PE_BASE_HXX

// USED SERVICES
    // BASE CLASSES
#include "cpp_pe.hxx"
    // COMPONENTS
#include <semantic/callf.hxx>
#include <semantic/sub_peu.hxx>
    // PARAMETERS
#include <ary/cpp/c_etypes.hxx>
// #include <ary/cpp/c_class.hxx>
#include <ary/cpp/c_idlist.hxx>

namespace cpp {

class PE_Type;

class PE_Base : public Cpp_PE
{
  public:
    enum E_State
    {
        startOfNext,
        inName,
        size_of_states
    };

    typedef ary::cpp::List_Bases        BaseList;
    typedef ary::cpp::S_Classes_Base    Base;

                        PE_Base(
                            Cpp_PE *            i_pParent );
                        ~PE_Base();

    const BaseList &    Result_BaseIds() const;

    virtual void        Call_Handler(
                            const cpp::Token &  i_rTok );

  private:
    typedef SubPe< PE_Base, PE_Type >           SP_Type;
    typedef SubPeUse< PE_Base, PE_Type>         SPU_BaseName;

    void                Setup_StatusFunctions();
    virtual void        InitData();
    virtual void        TransferData();
    void                Hdl_SyntaxError( const char *);

    void                SpReturn_BaseName();

    void                On_startOfNext_Identifier(const char *);
    void                On_startOfNext_public(const char *);
    void                On_startOfNext_protected(const char *);
    void                On_startOfNext_private(const char *);
    void                On_startOfNext_virtual(const char *);
    void                On_startOfNext_DoubleColon(const char *);

    void                On_inName_Identifier(const char *);
    void                On_inName_virtual(const char *);
    void                On_inName_SwBracket_Left(const char *);
    void                On_inName_DoubleColon(const char *);
    void                On_inName_Comma(const char *);

    Base &              CurObject();

    // DATA
    Dyn< PeStatusArray<PE_Base> >
                        pStati;

    Dyn<SP_Type>        pSpType;            /// till "{" incl.
    Dyn<SPU_BaseName>   pSpuBaseName;

    BaseList            aBaseIds;
};

// IMPLEMENTATION

inline const PE_Base::BaseList &
PE_Base::Result_BaseIds() const
    { return aBaseIds; }

inline PE_Base::Base &
PE_Base::CurObject()
    { return aBaseIds.back(); }

}   // namespace cpp

#endif

<commit_msg>INTEGRATION: CWS ooo19126 (1.1.1.1.146); FILE MERGED 2005/09/05 13:11:43 rt 1.1.1.1.146.1: #i54170# Change license header: remove SISSL<commit_after>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: pe_base.hxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: rt $ $Date: 2005-09-07 18:22:30 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef ADC_CPP_PE_BASE_HXX
#define ADC_CPP_PE_BASE_HXX

// USED SERVICES
    // BASE CLASSES
#include "cpp_pe.hxx"
    // COMPONENTS
#include <semantic/callf.hxx>
#include <semantic/sub_peu.hxx>
    // PARAMETERS
#include <ary/cpp/c_etypes.hxx>
// #include <ary/cpp/c_class.hxx>
#include <ary/cpp/c_idlist.hxx>

namespace cpp {

class PE_Type;

class PE_Base : public Cpp_PE
{
  public:
    enum E_State
    {
        startOfNext,
        inName,
        size_of_states
    };

    typedef ary::cpp::List_Bases        BaseList;
    typedef ary::cpp::S_Classes_Base    Base;

                        PE_Base(
                            Cpp_PE *            i_pParent );
                        ~PE_Base();

    const BaseList &    Result_BaseIds() const;

    virtual void        Call_Handler(
                            const cpp::Token &  i_rTok );

  private:
    typedef SubPe< PE_Base, PE_Type >           SP_Type;
    typedef SubPeUse< PE_Base, PE_Type>         SPU_BaseName;

    void                Setup_StatusFunctions();
    virtual void        InitData();
    virtual void        TransferData();
    void                Hdl_SyntaxError( const char *);

    void                SpReturn_BaseName();

    void                On_startOfNext_Identifier(const char *);
    void                On_startOfNext_public(const char *);
    void                On_startOfNext_protected(const char *);
    void                On_startOfNext_private(const char *);
    void                On_startOfNext_virtual(const char *);
    void                On_startOfNext_DoubleColon(const char *);

    void                On_inName_Identifier(const char *);
    void                On_inName_virtual(const char *);
    void                On_inName_SwBracket_Left(const char *);
    void                On_inName_DoubleColon(const char *);
    void                On_inName_Comma(const char *);

    Base &              CurObject();

    // DATA
    Dyn< PeStatusArray<PE_Base> >
                        pStati;

    Dyn<SP_Type>        pSpType;            /// till "{" incl.
    Dyn<SPU_BaseName>   pSpuBaseName;

    BaseList            aBaseIds;
};

// IMPLEMENTATION

inline const PE_Base::BaseList &
PE_Base::Result_BaseIds() const
    { return aBaseIds; }

inline PE_Base::Base &
PE_Base::CurObject()
    { return aBaseIds.back(); }

}   // namespace cpp

#endif

<|endoftext|>


<commit_before>/*
  StackThunk.c - Allow use second stack for BearSSL calls

  BearSSL uses a significant amount of stack space, much larger than
  the default Arduino core stack. These routines handle swapping
  between a secondary, user-allocated stack on the heap and the real
  stack.

  Copyright (c) 2017 Earle F. Philhower, III. All rights reserved.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  Modified 8 May 2015 by Hristo Gochkov (proper post and file upload handling)
*/

#include <stdint.h>
#include <stdlib.h>
#include "StackThunk.h"
#include <ets_sys.h>

extern "C" {

uint32_t *stack_thunk_ptr = NULL;
uint32_t *stack_thunk_top = NULL;
uint32_t *stack_thunk_save = NULL;  /* Saved A1 while in BearSSL */
uint32_t stack_thunk_refcnt = 0;

/* Largest stack usage seen in the wild at  6120 */
#define _stackSize (6200/4)
#define _stackPaint 0xdeadbeef

/* Add a reference, and allocate the stack if necessary */
void stack_thunk_add_ref()
{
  stack_thunk_refcnt++;
  if (stack_thunk_refcnt == 1) {
    stack_thunk_ptr = (uint32_t *)malloc(_stackSize * sizeof(uint32_t));
    stack_thunk_top = stack_thunk_ptr + _stackSize - 1;
    stack_thunk_save = NULL;
    stack_thunk_repaint();
  }
}

/* Drop a reference, and free stack if no more in use */
void stack_thunk_del_ref()
{
  if (stack_thunk_refcnt == 0) {
    /* Error! */
    return;
  }
  stack_thunk_refcnt--;
  if (!stack_thunk_refcnt) {
    free(stack_thunk_ptr);
    stack_thunk_ptr = NULL;
    stack_thunk_top = NULL;
    stack_thunk_save = NULL;
  }
}

void stack_thunk_repaint()
{
  for (int i=0; i < _stackSize; i++) {
    stack_thunk_ptr[i] = _stackPaint;
  }
}

/* Simple accessor functions used by postmortem */
uint32_t stack_thunk_get_refcnt() {
  return stack_thunk_refcnt;
}

uint32_t stack_thunk_get_stack_top() {
  return (uint32_t)stack_thunk_top;
}

uint32_t stack_thunk_get_stack_bot() {
  return (uint32_t)stack_thunk_ptr;
}

uint32_t stack_thunk_get_cont_sp() {
  return (uint32_t)stack_thunk_save;
}

/* Return the number of bytes ever used since the stack was created */
uint32_t stack_thunk_get_max_usage()
{
  uint32_t cnt = 0;

  /* No stack == no usage by definition! */
  if (!stack_thunk_ptr) {
    return 0;
  }

  for (cnt=0; (cnt < _stackSize) && (stack_thunk_ptr[cnt] == _stackPaint); cnt++) {
    /* Noop, all work done in for() */
  }
  return 4 * (_stackSize - cnt);
}

/* Print the stack from the first used 16-byte chunk to the top, decodable by the exception decoder */
void stack_thunk_dump_stack()
{
  uint32_t *pos = stack_thunk_ptr;
  while (pos < stack_thunk_top) {
    if ((pos[0] != _stackPaint) || (pos[1] != _stackPaint) || (pos[2] != _stackPaint) || (pos[3] != _stackPaint))
      break;
    pos += 4;
  }
  ets_printf(">>>stack>>>\n");
  while (pos < stack_thunk_top) {
    ets_printf("%08x:  %08x %08x %08x %08x\n", (int32_t)pos, pos[0], pos[1], pos[2], pos[3]);
    pos += 4;
  }
  ets_printf("<<<stack<<<\n");
}

/* Called when the stack overflow is detected by a thunk.  Main memory is corrupted at this point.  Do not return. */
void stack_thunk_fatal_overflow()
{
    ets_printf("FATAL ERROR: BSSL stack overflow\n");
    abort();
}

};
<commit_msg>Abort if BearSSL stack allocation fails (#7152)<commit_after>/*
  StackThunk.c - Allow use second stack for BearSSL calls

  BearSSL uses a significant amount of stack space, much larger than
  the default Arduino core stack. These routines handle swapping
  between a secondary, user-allocated stack on the heap and the real
  stack.

  Copyright (c) 2017 Earle F. Philhower, III. All rights reserved.

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  Modified 8 May 2015 by Hristo Gochkov (proper post and file upload handling)
*/

#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include "pgmspace.h"
#include "debug.h"
#include "StackThunk.h"
#include <ets_sys.h>

extern "C" {

uint32_t *stack_thunk_ptr = NULL;
uint32_t *stack_thunk_top = NULL;
uint32_t *stack_thunk_save = NULL;  /* Saved A1 while in BearSSL */
uint32_t stack_thunk_refcnt = 0;

/* Largest stack usage seen in the wild at  6120 */
#define _stackSize (6200/4)
#define _stackPaint 0xdeadbeef

/* Add a reference, and allocate the stack if necessary */
void stack_thunk_add_ref()
{
  stack_thunk_refcnt++;
  if (stack_thunk_refcnt == 1) {
    stack_thunk_ptr = (uint32_t *)malloc(_stackSize * sizeof(uint32_t));
    if (!stack_thunk_ptr) {
        // This is a fatal error, stop the sketch
        DEBUGV("Unable to allocate BearSSL stack\n");
        abort();
    }
    stack_thunk_top = stack_thunk_ptr + _stackSize - 1;
    stack_thunk_save = NULL;
    stack_thunk_repaint();
  }
}

/* Drop a reference, and free stack if no more in use */
void stack_thunk_del_ref()
{
  if (stack_thunk_refcnt == 0) {
    /* Error! */
    return;
  }
  stack_thunk_refcnt--;
  if (!stack_thunk_refcnt) {
    free(stack_thunk_ptr);
    stack_thunk_ptr = NULL;
    stack_thunk_top = NULL;
    stack_thunk_save = NULL;
  }
}

void stack_thunk_repaint()
{
  for (int i=0; i < _stackSize; i++) {
    stack_thunk_ptr[i] = _stackPaint;
  }
}

/* Simple accessor functions used by postmortem */
uint32_t stack_thunk_get_refcnt() {
  return stack_thunk_refcnt;
}

uint32_t stack_thunk_get_stack_top() {
  return (uint32_t)stack_thunk_top;
}

uint32_t stack_thunk_get_stack_bot() {
  return (uint32_t)stack_thunk_ptr;
}

uint32_t stack_thunk_get_cont_sp() {
  return (uint32_t)stack_thunk_save;
}

/* Return the number of bytes ever used since the stack was created */
uint32_t stack_thunk_get_max_usage()
{
  uint32_t cnt = 0;

  /* No stack == no usage by definition! */
  if (!stack_thunk_ptr) {
    return 0;
  }

  for (cnt=0; (cnt < _stackSize) && (stack_thunk_ptr[cnt] == _stackPaint); cnt++) {
    /* Noop, all work done in for() */
  }
  return 4 * (_stackSize - cnt);
}

/* Print the stack from the first used 16-byte chunk to the top, decodable by the exception decoder */
void stack_thunk_dump_stack()
{
  uint32_t *pos = stack_thunk_ptr;
  while (pos < stack_thunk_top) {
    if ((pos[0] != _stackPaint) || (pos[1] != _stackPaint) || (pos[2] != _stackPaint) || (pos[3] != _stackPaint))
      break;
    pos += 4;
  }
  ets_printf(">>>stack>>>\n");
  while (pos < stack_thunk_top) {
    ets_printf("%08x:  %08x %08x %08x %08x\n", (int32_t)pos, pos[0], pos[1], pos[2], pos[3]);
    pos += 4;
  }
  ets_printf("<<<stack<<<\n");
}

/* Called when the stack overflow is detected by a thunk.  Main memory is corrupted at this point.  Do not return. */
void stack_thunk_fatal_overflow()
{
    ets_printf("FATAL ERROR: BSSL stack overflow\n");
    abort();
}

};
<|endoftext|>


<commit_before>/** \file    Elasticsearch.cc
 *  \brief   Implementation of the Elasticsearch class.
 *  \author  Dr. Johannes Ruscheinski
 *  \author  Mario Trojan
 *
 *  \copyright 2018,2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "Elasticsearch.h"
#include "FileUtil.h"
#include "IniFile.h"
#include "UBTools.h"
#include "Url.h"

const std::string DEFAULT_CONFIG_FILE_PATH(UBTools::GetTuelibPath() + "Elasticsearch.conf");

static void LoadIniParameters(const std::string &config_file_path, std::string * const host, std::string * const username,
                              std::string * const password, bool * const ignore_ssl_certificates)
{
    if (not FileUtil::Exists(config_file_path))
        LOG_ERROR("Elasticsearch config file missing: " + config_file_path);

    const IniFile ini_file(config_file_path);

    *host                    = ini_file.getString("Elasticsearch", "host");
    *username                = ini_file.getString("Elasticsearch", "username", "");
    *password                = ini_file.getString("Elasticsearch", "password", "");
    *ignore_ssl_certificates = ini_file.getBool("Elasticsearch", "ignore_ssl_certificates", false);
}

Elasticsearch::Elasticsearch(const std::string &index, const std::string &type): index_(index), type_(type) {
    LoadIniParameters(DEFAULT_CONFIG_FILE_PATH, &host_, &username_, &password_, &ignore_ssl_certificates_);
}

size_t Elasticsearch::size() const {
    return JSON::LookupInteger("/_all/primaries/docs/count", query("_stats", REST::GET, JSON::ObjectNode()));
}

void Elasticsearch::simpleInsert(const std::map<std::string, std::string> &fields_and_values) {
    query("", REST::POST, JSON::ObjectNode(fields_and_values));
}

// A general comment as to the strategy we use in this function:
//
//    We know that there is an _update API endpoint, but as we insert a bunch of chunks, the number of which can change,
//    we have to use a "wildcard" strategy to first delete possibly existing chunks as the number of new chunks may be greater or
//    fewer than what may already be stored.
void Elasticsearch::insertOrUpdateDocument(const std::string &document_id, const std::string &document) {
    deleteDocument(document_id);
    simpleInsert({ { "id", document_id }, { "document", document } });
}

bool Elasticsearch::deleteDocument(const std::string &document_id) {
    const JSON::ObjectNode match_node("{ \"query\":"
                                      "    { \"match\":"
                                      "        { \"id\": \"" + JSON::EscapeString(document_id) + "\" }"
                                      "    }"
                                      "}");
    const auto result_node(query("_delete_by_query", REST::POST, match_node));
    return result_node->getIntegerNode("deleted")->getValue() > 0;
}

std::unordered_set<std::string> Elasticsearch::selectAll(const std::string &field) const {
    const std::vector<std::map<std::string, std::string>> result(simpleSelect({ field }));
    std::unordered_set<std::string> unique_values;
    for (const auto &map : result) {
        const auto key_and_value(map.find(field));
        if (key_and_value != map.cend())
            unique_values.emplace(key_and_value->second);
    }

    return unique_values;
}

std::vector<std::map<std::string, std::string>> Elasticsearch::simpleSelect(const std::set<std::string> &fields,
                                                                            const std::map<std::string, std::string> &filter,
                                                                            const int max_count) const
{
    std::string query_string("{\n");

    if (not fields.empty()) {
        query_string += "    \"_source\": [";
        for (const auto &field : fields)
            query_string += "\"" + field + "\", ";
        query_string.resize(query_string.size() - 2); // Remove trailing comma and space.
        query_string += "],\n";
    }

    query_string += "    \"query\": {";

    if (filter.empty())
        query_string += " \"match_all\" {}";
    else {
        query_string += "\"bool\" : { \"filter\": [\n";
        for (const auto &field_and_value : filter)
            query_string += "       { \"term\": { \"" + field_and_value.first + "\": \"" + field_and_value.second + "\"} },\n";
        query_string.resize(query_string.size() - 2); // Remove the last comma and newline.
        query_string += "\n    ] }";
    }

    query_string += "    },\n";
    query_string += "    \"size\": " + std::to_string(max_count) + "\n";
    query_string += "}\n";

    const auto result_node(query("_search", REST::POST, JSON::ObjectNode(query_string)));
    const auto hits_object_node(result_node->getObjectNode("hits"));
    if (unlikely(hits_object_node == nullptr))
        LOG_ERROR("missing \"hits\" object node in Elasticsearch result node!");
    const auto hits_array_node(hits_object_node->getArrayNode("hits"));
    if (unlikely(hits_array_node == nullptr))
        LOG_ERROR("missing \"hits\" array node in Elasticsearch result node!");

    std::vector<std::map<std::string, std::string>> search_results;
    search_results.reserve(hits_array_node->size());
    for (const auto entry_node : *hits_array_node) {
        std::map<std::string, std::string> new_map;
        const auto entry_object_node(JSON::JSONNode::CastToObjectNodeOrDie("entry_object_node", entry_node));
        for (const auto &entry : *entry_object_node) {
            if (fields.empty() or fields.find(entry.first) != fields.cend())
                new_map[entry.first] = JSON::JSONNode::CastToStringNodeOrDie("new_map[entry.first]", entry.second)->getValue();
        }

        search_results.resize(search_results.size() + 1);
        std::swap(search_results.back(), new_map);
    }

    return search_results;
}

static std::string ToString(const Elasticsearch::RangeOperator op) {
    switch (op) {
    case Elasticsearch::RO_GT:  return "gt";
    case Elasticsearch::RO_GTE: return "gte";
    case Elasticsearch::RO_LT:  return "lt";
    case Elasticsearch::RO_LTE: return "lte";
    default:
        LOG_ERROR("we should *never* get here!");
    }
}

bool Elasticsearch::deleteRange(const std::string &field, const RangeOperator operator1, const std::string &operand1,
                                const RangeOperator operator2, const std::string &operand2)
{
    const std::string range_node((operator2 == RO_NOOP or operand2.empty()) ?
                                    "{ \"query\":"
                                    "    { \"range\":"
                                    "        \"" + field + "\": {"
                                    "            \"" + ToString(operator1) + "\"" + operand1 + "\""
                                        + ((operator2 == RO_NOOP or operand2.empty())
                                               ? std::string()
                                               : "      ,\"" + ToString(operator2) + "\"" + operand2 + "\"") +
                                    "        }"
                                    "    }"
                                    "}"
                                                                            :
                                    "");
    const auto result_node(query("_delete", REST::POST, range_node));
    return result_node->getIntegerNode("deleted")->getValue() > 0;
}

bool Elasticsearch::fieldWithValueExists(const std::string &field, const std::string &value) {

   const auto result_node(
         query("_search", REST::POST,
              "{"
              "    \"query\": {"
              "        \"match\" : { \"" + field + "\" : \"" + value + "\" }"
              "    }"
              "}"
         ));

   const auto hits_node(result_node->getObjectNode("hits"));
   if (hits_node == nullptr)
       LOG_ERROR("No \"hits\" node in results");

   const auto total_node = hits_node->getIntegerNode("total");
   if (total_node == nullptr)
       LOG_ERROR("No \"total\" node found");
   return total_node->getValue() == 0 ? false : true;
}

std::shared_ptr<JSON::ObjectNode> Elasticsearch::query(const std::string &action, const REST::QueryType query_type,
                                                       const JSON::ObjectNode &data) const
{
    Downloader::Params downloader_params;
    downloader_params.authentication_username_ = username_;
    downloader_params.authentication_password_ = password_;
    downloader_params.ignore_ssl_certificates_ = ignore_ssl_certificates_;
    downloader_params.additional_headers_.push_back("Content-Type: application/json");
    const Url url(host_ + "/" + index_ + "/" + type_ + (action.empty() ? "" : "/" + action));
    std::shared_ptr<JSON::JSONNode> result(REST::QueryJSON(url, query_type, &data, downloader_params));
    std::shared_ptr<JSON::ObjectNode> result_object(JSON::JSONNode::CastToObjectNodeOrDie("Elasticsearch result", result));
    if (result_object->hasNode("error"))
        LOG_ERROR("Elasticsearch " + action + " query failed: " + result_object->getNode("error")->toString());

    return result_object;
}
<commit_msg>Flatten _source field<commit_after>/** \file    Elasticsearch.cc
 *  \brief   Implementation of the Elasticsearch class.
 *  \author  Dr. Johannes Ruscheinski
 *  \author  Mario Trojan
 *
 *  \copyright 2018,2019 Universitätsbibliothek Tübingen.  All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU Affero General Public License as
 *  published by the Free Software Foundation, either version 3 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Affero General Public License for more details.
 *
 *  You should have received a copy of the GNU Affero General Public License
 *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "Elasticsearch.h"
#include "FileUtil.h"
#include "IniFile.h"
#include "UBTools.h"
#include "Url.h"

const std::string DEFAULT_CONFIG_FILE_PATH(UBTools::GetTuelibPath() + "Elasticsearch.conf");

static void LoadIniParameters(const std::string &config_file_path, std::string * const host, std::string * const username,
                              std::string * const password, bool * const ignore_ssl_certificates)
{
    if (not FileUtil::Exists(config_file_path))
        LOG_ERROR("Elasticsearch config file missing: " + config_file_path);

    const IniFile ini_file(config_file_path);

    *host                    = ini_file.getString("Elasticsearch", "host");
    *username                = ini_file.getString("Elasticsearch", "username", "");
    *password                = ini_file.getString("Elasticsearch", "password", "");
    *ignore_ssl_certificates = ini_file.getBool("Elasticsearch", "ignore_ssl_certificates", false);
}

Elasticsearch::Elasticsearch(const std::string &index, const std::string &type): index_(index), type_(type) {
    LoadIniParameters(DEFAULT_CONFIG_FILE_PATH, &host_, &username_, &password_, &ignore_ssl_certificates_);
}

size_t Elasticsearch::size() const {
    return JSON::LookupInteger("/_all/primaries/docs/count", query("_stats", REST::GET, JSON::ObjectNode()));
}

void Elasticsearch::simpleInsert(const std::map<std::string, std::string> &fields_and_values) {
    query("", REST::POST, JSON::ObjectNode(fields_and_values));
}

// A general comment as to the strategy we use in this function:
//
//    We know that there is an _update API endpoint, but as we insert a bunch of chunks, the number of which can change,
//    we have to use a "wildcard" strategy to first delete possibly existing chunks as the number of new chunks may be greater or
//    fewer than what may already be stored.
void Elasticsearch::insertOrUpdateDocument(const std::string &document_id, const std::string &document) {
    deleteDocument(document_id);
    simpleInsert({ { "id", document_id }, { "document", document } });
}

bool Elasticsearch::deleteDocument(const std::string &document_id) {
    const JSON::ObjectNode match_node("{ \"query\":"
                                      "    { \"match\":"
                                      "        { \"id\": \"" + JSON::EscapeString(document_id) + "\" }"
                                      "    }"
                                      "}");
    const auto result_node(query("_delete_by_query", REST::POST, match_node));
    return result_node->getIntegerNode("deleted")->getValue() > 0;
}

std::unordered_set<std::string> Elasticsearch::selectAll(const std::string &field) const {
    const std::vector<std::map<std::string, std::string>> result(simpleSelect({ field }));
    std::unordered_set<std::string> unique_values;
    for (const auto &map : result) {
        const auto key_and_value(map.find(field));
        if (key_and_value != map.cend())
            unique_values.emplace(key_and_value->second);
    }

    return unique_values;
}

std::vector<std::map<std::string, std::string>> Elasticsearch::simpleSelect(const std::set<std::string> &fields,
                                                                            const std::map<std::string, std::string> &filter,
                                                                            const int max_count) const
{
    std::string query_string("{\n");

    if (not fields.empty()) {
        query_string += "    \"_source\": [";
        for (const auto &field : fields)
            query_string += "\"" + field + "\", ";
        query_string.resize(query_string.size() - 2); // Remove trailing comma and space.
        query_string += "],\n";
    }

    query_string += "    \"query\": {";

    if (filter.empty())
        query_string += " \"match_all\" {}";
    else {
        query_string += "\"bool\" : { \"filter\": [\n";
        for (const auto &field_and_value : filter)
            query_string += "       { \"term\": { \"" + field_and_value.first + "\": \"" + field_and_value.second + "\"} },\n";
        query_string.resize(query_string.size() - 2); // Remove the last comma and newline.
        query_string += "\n    ] }";
    }

    query_string += "    },\n";
    query_string += "    \"size\": " + std::to_string(max_count) + "\n";
    query_string += "}\n";

    const auto result_node(query("_search", REST::POST, JSON::ObjectNode(query_string)));
    const auto hits_object_node(result_node->getObjectNode("hits"));
    if (unlikely(hits_object_node == nullptr))
        LOG_ERROR("missing \"hits\" object node in Elasticsearch result node!");
    const auto hits_array_node(hits_object_node->getArrayNode("hits"));
    if (unlikely(hits_array_node == nullptr))
        LOG_ERROR("missing \"hits\" array node in Elasticsearch result node!");

    std::vector<std::map<std::string, std::string>> search_results;
    search_results.reserve(hits_array_node->size());
    for (const auto entry_node : *hits_array_node) {
        std::map<std::string, std::string> new_map;
        const auto entry_object_node(JSON::JSONNode::CastToObjectNodeOrDie("entry_object_node", entry_node));
        for (const auto &entry : *entry_object_node) {
            if (fields.empty() or fields.find(entry.first) != fields.cend() or entry.first == "_source") {
                // Copy existing fields but flatten the contents of _source
                if (entry.first == "_source") {
                    const auto source_object_node(JSON::JSONNode::CastToObjectNodeOrDie("source_object_node", entry.second));
                    for (const auto source_entry : *source_object_node)
                         new_map[source_entry.first] = JSON::JSONNode::CastToStringNodeOrDie("new_map[source_entry.first]",
                                                                                source_entry.second)->getValue();
                } else
                    new_map[entry.first] = JSON::JSONNode::CastToStringNodeOrDie("new_map[entry.first]", entry.second)->getValue();
            }
        }

        search_results.resize(search_results.size() + 1);
        std::swap(search_results.back(), new_map);
    }

    return search_results;
}

static std::string ToString(const Elasticsearch::RangeOperator op) {
    switch (op) {
    case Elasticsearch::RO_GT:  return "gt";
    case Elasticsearch::RO_GTE: return "gte";
    case Elasticsearch::RO_LT:  return "lt";
    case Elasticsearch::RO_LTE: return "lte";
    default:
        LOG_ERROR("we should *never* get here!");
    }
}

bool Elasticsearch::deleteRange(const std::string &field, const RangeOperator operator1, const std::string &operand1,
                                const RangeOperator operator2, const std::string &operand2)
{
    const std::string range_node((operator2 == RO_NOOP or operand2.empty()) ?
                                    "{ \"query\":"
                                    "    { \"range\":"
                                    "        \"" + field + "\": {"
                                    "            \"" + ToString(operator1) + "\"" + operand1 + "\""
                                        + ((operator2 == RO_NOOP or operand2.empty())
                                               ? std::string()
                                               : "      ,\"" + ToString(operator2) + "\"" + operand2 + "\"") +
                                    "        }"
                                    "    }"
                                    "}"
                                                                            :
                                    "");
    const auto result_node(query("_delete", REST::POST, range_node));
    return result_node->getIntegerNode("deleted")->getValue() > 0;
}

bool Elasticsearch::fieldWithValueExists(const std::string &field, const std::string &value) {

   const auto result_node(
         query("_search", REST::POST,
              "{"
              "    \"query\": {"
              "        \"match\" : { \"" + field + "\" : \"" + value + "\" }"
              "    }"
              "}"
         ));

   const auto hits_node(result_node->getObjectNode("hits"));
   if (hits_node == nullptr)
       LOG_ERROR("No \"hits\" node in results");

   const auto total_node = hits_node->getIntegerNode("total");
   if (total_node == nullptr)
       LOG_ERROR("No \"total\" node found");
   return total_node->getValue() == 0 ? false : true;
}

std::shared_ptr<JSON::ObjectNode> Elasticsearch::query(const std::string &action, const REST::QueryType query_type,
                                                       const JSON::ObjectNode &data) const
{
    Downloader::Params downloader_params;
    downloader_params.authentication_username_ = username_;
    downloader_params.authentication_password_ = password_;
    downloader_params.ignore_ssl_certificates_ = ignore_ssl_certificates_;
    downloader_params.additional_headers_.push_back("Content-Type: application/json");
    const Url url(host_ + "/" + index_ + "/" + type_ + (action.empty() ? "" : "/" + action));
    std::shared_ptr<JSON::JSONNode> result(REST::QueryJSON(url, query_type, &data, downloader_params));
    std::shared_ptr<JSON::ObjectNode> result_object(JSON::JSONNode::CastToObjectNodeOrDie("Elasticsearch result", result));
    if (result_object->hasNode("error"))
        LOG_ERROR("Elasticsearch " + action + " query failed: " + result_object->getNode("error")->toString());

    return result_object;
}
<|endoftext|>


<commit_before>#include "node.hpp"
#include "frequency_map.hpp"
#include <map>
#include <fstream>
#include <queue>
#include <functional>
#include <string>

#ifndef HUFFMAN_TREE_HPP
#define HUFFMAN_TREE_HPP

namespace cpr
{
	namespace huffman
	{
		template<typename Char, typename Freq>
		class HuffmanTree
		{
		public:
			using SharedNode = typename Node<Char, Freq>::SharedNode;

			explicit HuffmanTree(FrequencyMap<Char, Freq> const& map)
				: root{make_tree(map)}
			{}

			std::string to_string() const
			{
				std::string str;
				std::function<void(SharedNode)> inorder = [&](SharedNode node)
				{
					if (node)
					{
						inorder(node->left_);

						str.push_back('[');
						str.push_back(node->character_ == 0 ? '_' : node->character_);
						str.push_back(',');
						str += std::to_string(node->freq_);
						str.push_back(']');

						inorder(node->right_);
					}
				};

				inorder(root);
				return str;
			}

			const SharedNode root;

		private:
			// huffman coding algorithm
			// based on a pseudocode on 16.3 CLRS 3rd.
			SharedNode make_tree(FrequencyMap<Char, Freq> const& map) const
			{
					auto greater = [](SharedNode lhs, SharedNode rhs)
					{
						if (lhs->freq_ != rhs->freq_)
							return lhs->freq_ > rhs->freq_;
						else
							return lhs->character_ > rhs->character_;
					};

					using MinPriorityQueue = std::priority_queue < SharedNode, std::vector<SharedNode>, decltype(greater) > ;

					MinPriorityQueue queue(greater);
					for (auto const& pair : map)
						queue.push(make_new_node(pair.first, pair.second));

					for (int count = 1; count != map.size(); ++count)
					{
						auto merge = make_new_node<Char, Freq>();
						merge->left_ = make_new_node(*queue.top());
						queue.pop();
						merge->right_ = make_new_node(*queue.top());
						queue.pop();
						merge->freq_ = merge->left_->freq_ + merge->right_->freq_;
						queue.push(merge);
					}

					return queue.top();
			}
		};
	}
}

#endif // !HUFFMAN_TREE_HPP<commit_msg>fixed #8 	modified:   cpr/huffman/huffman_tree.hpp<commit_after>#include "node.hpp"
#include "frequency_map.hpp"
#include <map>
#include <fstream>
#include <queue>
#include <functional>
#include <string>

#ifndef HUFFMAN_TREE_HPP
#define HUFFMAN_TREE_HPP

namespace cpr
{
	namespace huffman
	{
		template<typename Char, typename Freq>
		class HuffmanTree
		{
		public:
			using SharedNode = typename Node<Char, Freq>::SharedNode;

			explicit HuffmanTree(FrequencyMap<Char, Freq> const& map)
				: root{make_tree(map)}
			{}

			std::string to_string() const
			{
				std::string str;
				std::function<void(SharedNode)> inorder = [&](SharedNode node)
				{
					if (node)
					{
						inorder(node->left_);

						str.push_back('[');
						str.push_back(node->character_ == 0 ? '_' : node->character_);
						str.push_back(',');
						str += std::to_string(node->freq_);
						str.push_back(']');

						inorder(node->right_);
					}
				};

				inorder(root);
				return str;
			}

			const SharedNode root;

		private:
			// huffman coding algorithm
			// based on a pseudocode on 16.3 CLRS 3rd.
			SharedNode make_tree(FrequencyMap<Char, Freq> const& map) const
			{
					auto greater = [](SharedNode lhs, SharedNode rhs)
					{
						if (lhs->freq_ != rhs->freq_)
							return lhs->freq_ > rhs->freq_;
						else
							return lhs->character_ > rhs->character_;
					};

					using MinPriorityQueue = std::priority_queue < SharedNode, std::vector<SharedNode>, decltype(greater) > ;

					MinPriorityQueue queue(greater);
					for (auto const& pair : map)
						queue.push(make_new_node(pair.first, pair.second));

					for (int count = 1; count != map.size(); ++count)
					{
						auto merge = make_new_node<Char, Freq>();
						merge->left_ = queue.top();		queue.pop();
						merge->right_ = queue.top();	queue.pop();
						merge->freq_ = merge->left_->freq_ + merge->right_->freq_;
						queue.push(merge);
					}

					return queue.top();
			}
		};
	}
}

#endif // !HUFFMAN_TREE_HPP<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <memory>
#include <string>
#include <vector>

#include "tensorflow/compiler/xla/layout_util.h"
#include "tensorflow/compiler/xla/literal_util.h"
#include "tensorflow/compiler/xla/service/device_memory_allocator.h"
#include "tensorflow/compiler/xla/service/generic_transfer_manager.h"
#include "tensorflow/compiler/xla/service/shaped_buffer.h"
#include "tensorflow/compiler/xla/shape_util.h"
#include "tensorflow/compiler/xla/statusor.h"
#include "tensorflow/compiler/xla/tests/literal_test_util.h"
#include "tensorflow/compiler/xla/tests/local_client_test_base.h"
#include "tensorflow/compiler/xla/tests/test_macros.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/stream_executor_no_cuda.h"
#include "tensorflow/core/platform/types.h"

namespace se = ::perftools::gputools;

namespace xla {

namespace {

class TransferManagerTest : public LocalClientTestBase {
 protected:
  TransferManagerTest() {
    shape_size_fn_ = [this](const Shape& shape) {
      return transfer_manager_->GetByteSizeRequirement(shape);
    };
  }

  ~TransferManagerTest() override {}

  std::unique_ptr<ScopedShapedBuffer> AllocateDeviceBuffer(const Shape& shape) {
    return ScopedShapedBuffer::Allocate(
               shape, GetOrCreateAllocator(local_client_->platform()),
               /*device_ordinal=*/0, shape_size_fn_)
        .ConsumeValueOrDie();
  }

  std::function<int64(const Shape&)> shape_size_fn_;
};

XLA_TEST_F(TransferManagerTest, TransferR0U32) {
  std::unique_ptr<Literal> literal = Literal::CreateR0<uint32>(42);
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR0Equal<uint32>(42, *result);
}

XLA_TEST_F(TransferManagerTest, TransferR1F32) {
  std::unique_ptr<Literal> literal =
      Literal::CreateR1<float>({1.25f, 2.5f, -17.0f, -20.125f});
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR1Equal<float>({1.25f, 2.5f, -17.0f, -20.125f},
                                        *result);
}

XLA_TEST_F(TransferManagerTest, TransferR1LargeF32) {
  std::vector<float> test_vector(1024 * 1024);
  std::iota(test_vector.begin(), test_vector.end(), 0);
  std::unique_ptr<Literal> literal = Literal::CreateR1<float>(test_vector);
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR1Equal<float>(test_vector, *result);
}

XLA_TEST_F(TransferManagerTest, TransferR1U8) {
  const char* test_string = "0123456789abcdef";
  std::unique_ptr<Literal> literal = Literal::CreateR1U8(test_string);
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  EXPECT_EQ(result->u8s_string(), test_string);
}

XLA_TEST_F(TransferManagerTest, TransferR2F32) {
  std::unique_ptr<Literal> literal =
      Literal::CreateR2<float>({{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}});
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR2Equal<float>(
      {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, *result);
}

XLA_TEST_F(TransferManagerTest,
           TransferR2F32AndChangeLayoutTransferringToDevice) {
  std::unique_ptr<Literal> literal = Literal::CreateR2WithLayout<float>(
      {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, LayoutUtil::MakeLayout({0, 1}));
  const Shape ondevice_shape =
      ShapeUtil::MakeShapeWithLayout(F32, {2, 3}, {1, 0});
  auto device_buffer = AllocateDeviceBuffer(ondevice_shape);

  // Round trip literal through device. Set the on-device layout to something
  // different than the literal layout.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  EXPECT_FALSE(
      LayoutUtil::Equal(result->shape().layout(), literal->shape().layout()));
  LiteralTestUtil::ExpectR2Equal<float>(
      {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, *result);
}

XLA_TEST_F(TransferManagerTest, TransferTuple) {
  std::unique_ptr<Literal> literal = Literal::MakeTuple(
      {Literal::CreateR0<float>(123.0f).get(),
       Literal::CreateR2<float>({{1.0f, 2.0f}, {4.0f, 5.0f}}).get(),
       Literal::CreateR1<float>({44.0f, -10.0f, 3333333.3f}).get()});
  auto device_buffer = AllocateDeviceBuffer(literal->shape());

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectEqual(*literal, *result);
}

XLA_TEST_F(TransferManagerTest, TransferEmptyTuple) {
  std::unique_ptr<Literal> literal = Literal::MakeTuple({});
  auto device_buffer = AllocateDeviceBuffer(literal->shape());

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectEqual(*literal, *result);
}

XLA_TEST_F(TransferManagerTest, TransferNestedTuple) {
  std::unique_ptr<Literal> literal = Literal::MakeTuple(
      {Literal::CreateR0<float>(123.0f).get(),
       Literal::MakeTuple(
           {Literal::CreateR2<float>({{1.0f, 2.0f}, {4.0f, 5.0f}}).get(),
            Literal::CreateR1<float>({44.0f, -10.0f, 3333333.3f}).get()})
           .get(),
       Literal::CreateR1<float>({-10.0f, 123.0f}).get()});
  auto device_buffer = AllocateDeviceBuffer(literal->shape());

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectEqual(*literal, *result);
}

}  // namespace

}  // namespace xla
<commit_msg>Clean up initialization, declarations and access of TransferManagerTest<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include <memory>
#include <string>
#include <vector>

#include "tensorflow/compiler/xla/layout_util.h"
#include "tensorflow/compiler/xla/literal_util.h"
#include "tensorflow/compiler/xla/service/device_memory_allocator.h"
#include "tensorflow/compiler/xla/service/generic_transfer_manager.h"
#include "tensorflow/compiler/xla/service/shaped_buffer.h"
#include "tensorflow/compiler/xla/shape_util.h"
#include "tensorflow/compiler/xla/statusor.h"
#include "tensorflow/compiler/xla/tests/literal_test_util.h"
#include "tensorflow/compiler/xla/tests/local_client_test_base.h"
#include "tensorflow/compiler/xla/tests/test_macros.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/compiler/xla/xla_data.pb.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/stream_executor_no_cuda.h"
#include "tensorflow/core/platform/types.h"

namespace xla {
namespace {

class TransferManagerTest : public LocalClientTestBase {
 protected:
  TransferManagerTest()
      : shape_size_fn_([this](const Shape& shape) {
          return transfer_manager_->GetByteSizeRequirement(shape);
        }) {}

  ~TransferManagerTest() override = default;

  std::unique_ptr<ScopedShapedBuffer> AllocateDeviceBuffer(const Shape& shape) {
    return ScopedShapedBuffer::Allocate(
               shape, GetOrCreateAllocator(local_client_->platform()),
               /*device_ordinal=*/0, shape_size_fn_)
        .ValueOrDie();
  }

 private:
  std::function<int64(const Shape&)> shape_size_fn_;
};

XLA_TEST_F(TransferManagerTest, TransferR0U32) {
  std::unique_ptr<Literal> literal = Literal::CreateR0<uint32>(42);
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR0Equal<uint32>(42, *result);
}

XLA_TEST_F(TransferManagerTest, TransferR1F32) {
  std::unique_ptr<Literal> literal =
      Literal::CreateR1<float>({1.25f, 2.5f, -17.0f, -20.125f});
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR1Equal<float>({1.25f, 2.5f, -17.0f, -20.125f},
                                        *result);
}

XLA_TEST_F(TransferManagerTest, TransferR1LargeF32) {
  std::vector<float> test_vector(1024 * 1024);
  std::iota(test_vector.begin(), test_vector.end(), 0);
  std::unique_ptr<Literal> literal = Literal::CreateR1<float>(test_vector);
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR1Equal<float>(test_vector, *result);
}

XLA_TEST_F(TransferManagerTest, TransferR1U8) {
  const char* test_string = "0123456789abcdef";
  std::unique_ptr<Literal> literal = Literal::CreateR1U8(test_string);
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  EXPECT_EQ(result->u8s_string(), test_string);
}

XLA_TEST_F(TransferManagerTest, TransferR2F32) {
  std::unique_ptr<Literal> literal =
      Literal::CreateR2<float>({{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}});
  const Shape& shape = literal->shape();
  auto device_buffer = AllocateDeviceBuffer(shape);

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectR2Equal<float>(
      {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, *result);
}

XLA_TEST_F(TransferManagerTest,
           TransferR2F32AndChangeLayoutTransferringToDevice) {
  std::unique_ptr<Literal> literal = Literal::CreateR2WithLayout<float>(
      {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, LayoutUtil::MakeLayout({0, 1}));
  const Shape ondevice_shape =
      ShapeUtil::MakeShapeWithLayout(F32, {2, 3}, {1, 0});
  auto device_buffer = AllocateDeviceBuffer(ondevice_shape);

  // Round trip literal through device. Set the on-device layout to something
  // different than the literal layout.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  EXPECT_FALSE(
      LayoutUtil::Equal(result->shape().layout(), literal->shape().layout()));
  LiteralTestUtil::ExpectR2Equal<float>(
      {{1.0f, 2.0f, 3.0f}, {4.0f, 5.0f, 6.0f}}, *result);
}

XLA_TEST_F(TransferManagerTest, TransferTuple) {
  std::unique_ptr<Literal> literal = Literal::MakeTuple(
      {Literal::CreateR0<float>(123.0f).get(),
       Literal::CreateR2<float>({{1.0f, 2.0f}, {4.0f, 5.0f}}).get(),
       Literal::CreateR1<float>({44.0f, -10.0f, 3333333.3f}).get()});
  auto device_buffer = AllocateDeviceBuffer(literal->shape());

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectEqual(*literal, *result);
}

XLA_TEST_F(TransferManagerTest, TransferEmptyTuple) {
  std::unique_ptr<Literal> literal = Literal::MakeTuple({});
  auto device_buffer = AllocateDeviceBuffer(literal->shape());

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectEqual(*literal, *result);
}

XLA_TEST_F(TransferManagerTest, TransferNestedTuple) {
  std::unique_ptr<Literal> literal = Literal::MakeTuple(
      {Literal::CreateR0<float>(123.0f).get(),
       Literal::MakeTuple(
           {Literal::CreateR2<float>({{1.0f, 2.0f}, {4.0f, 5.0f}}).get(),
            Literal::CreateR1<float>({44.0f, -10.0f, 3333333.3f}).get()})
           .get(),
       Literal::CreateR1<float>({-10.0f, 123.0f}).get()});
  auto device_buffer = AllocateDeviceBuffer(literal->shape());

  // Round trip literal through device.
  ASSERT_IS_OK(transfer_manager_->TransferLiteralToDevice(
      stream_executor_, *literal, *device_buffer));
  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
                          transfer_manager_->TransferLiteralFromDevice(
                              stream_executor_, *device_buffer));

  LiteralTestUtil::ExpectEqual(*literal, *result);
}

}  // namespace
}  // namespace xla
<|endoftext|>


<commit_before>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/lite/delegates/gpu/metal/kernels/padding.h"

#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "absl/strings/substitute.h"
#include "tensorflow/lite/delegates/gpu/common/model.h"
#include "tensorflow/lite/delegates/gpu/common/operations.h"
#include "tensorflow/lite/delegates/gpu/common/types.h"
#include "tensorflow/lite/delegates/gpu/common/util.h"
#include "tensorflow/lite/delegates/gpu/metal/compute_task_descriptor.h"

namespace tflite {
namespace gpu {
namespace metal {
namespace {

std::string GetPaddingCode(const PadAttributes& attr) {
  const std::string channels[] = {".x", ".y", ".z", ".w"};
  std::string code = R"(
    #include <metal_stdlib>
    using namespace metal;

    struct uniforms {
      int4 src_size;
      int4 dst_size;
      int4 padding;
    };)";
  if (attr.type == PaddingContentType::REFLECT) {
    code += R"(
    int reflect(int x, int size) {
      return size - 1 - abs(abs(x) - size + 1);
    })";
  }
  code += R"(
    $0
    kernel void ComputeFunction(
                                $1
                                uint3 gid[[thread_position_in_grid]]) {
      if (static_cast<int>(gid.x) >= params.dst_size.x ||
          static_cast<int>(gid.y) >= params.dst_size.y) {
        return;
      }

      FLT4 value = FLT4(0.0f);
      int s_x = static_cast<int>(gid.x) - params.padding.x;
      int s_y = static_cast<int>(gid.y) - params.padding.y;)";
  if (attr.type == PaddingContentType::REFLECT) {
    code += R"(
      s_x = reflect(s_x, params.src_size.x);
      s_y = reflect(s_y, params.src_size.y);
)";
    if (attr.prepended.c == 0 && attr.appended.c == 0) {
      // optimized case
      code +=
          "      int buffer_index = (int(gid.z) * params.src_size.y + s_y) * "
          "params.src_size.x + s_x;\n";
      code += "      value = src_tensor[buffer_index];\n";
    } else {
      code += "      int start_channel = static_cast<int>(gid.z) * 4;\n";
      for (int i = 0; i < 4; ++i) {
        const auto& s = channels[i];
        code += "      {\n";
        code += "        int channel = start_channel + " + std::to_string(i) +
                ";\n";
        code += "        int s_z = channel - params.padding.z;\n";
        // We need additional clamp for z, so that we use alignment for channels
        // and can proceed extra channels that can lead to reading out of
        // resource.
        code +=
            "        s_z = clamp(reflect(s_z, params.src_size.z), 0, "
            "params.src_size.z - 1);\n";
        code +=
            "        int buffer_index = ((s_z / 4) * params.src_size.y + s_y) "
            "* params.src_size.x + s_x;\n";
        code += "        FLT4 t = src_tensor[buffer_index];\n";
        code += "        FLT t_ar[4] = {t.x, t.y, t.z, t.w};\n";
        code += "        value" + s + " = t_ar[s_z % 4];\n";
        code += "      }\n";
      }
    }
  } else {
    code += R"(
      bool inside_x = s_x >= 0 && s_x < params.src_size.x;
      bool inside_y = s_y >= 0 && s_y < params.src_size.y;
      if (inside_x && inside_y) {
        int start_channel = static_cast<int>(gid.z) * 4;
    )";
    if (attr.prepended.c == 0 && attr.appended.c == 0) {
      // optimized case
      code +=
          "        int buffer_index = (int(gid.z) * params.src_size.y + s_y) * "
          "params.src_size.x + s_x;\n";
      code += "        value = src_tensor[buffer_index];\n";
    } else if (attr.prepended.c % 4 == 0) {
      code += R"(
        int s_z = static_cast<int>(gid.z) - params.padding.z / 4;
        if (s_z >= 0 && s_z < params.src_size.w) {
          int buffer_index = (s_z * params.src_size.y + s_y) * params.src_size.x + s_x;
          value = src_tensor[buffer_index];
        })";
    } else {
      for (int i = 0; i < 4; ++i) {
        const auto& s = channels[i];
        code += "    {\n";
        code +=
            "    int channel = start_channel + " + std::to_string(i) + ";\n";
        code += "    int s_z = channel - params.padding.z;\n";
        code += "    if (s_z >= 0 && s_z < params.src_size.z) {\n";
        code +=
            "      int buffer_index = ((s_z / 4) * params.src_size.y + s_y) * "
            "params.src_size.x + "
            "s_x;\n";
        code += "      FLT4 t = src_tensor[buffer_index];\n";
        code += "      FLT t_ar[4] = {t.x, t.y, t.z, t.w};\n";
        code += "      value" + s + " = t_ar[s_z % 4];\n";
        code += "    }\n";
        code += "    }\n";
      }
    }
    code += "  }\n";
  }
  code +=
      "  int linear_index = (gid.z * params.dst_size.y + int(gid.y)) * "
      "params.dst_size.x + "
      "int(gid.x);\n";
  code += "  $2\n";
  code += "  dst_tensor[linear_index] = value;\n";
  code += "}\n";
  return code;
}
}  // namespace

ComputeTaskDescriptor Padding(const OperationDef& definition,
                              const PadAttributes& attr) {
  ComputeTaskDescriptor desc(definition);
  desc.shader_source = GetPaddingCode(attr);

  desc.AddSrcTensor("src_tensor", definition.src_tensors[0]);
  desc.AddDstTensor("dst_tensor", definition.dst_tensors[0]);

  desc.uniform_buffers = {
      {"constant uniforms& params",
       [attr](const std::vector<BHWC>& src_shapes,
              const std::vector<BHWC>& dst_shapes) {
         std::vector<int> uniform_params{
             // int4 src_size
             src_shapes[0].w,
             src_shapes[0].h,
             src_shapes[0].c,
             DivideRoundUp(src_shapes[0].c, 4),
             // int4 dst_size
             dst_shapes[0].w,
             dst_shapes[0].h,
             dst_shapes[0].c,
             DivideRoundUp(dst_shapes[0].c, 4),
             // int4 prepended padding
             attr.prepended.w,
             attr.prepended.h,
             attr.prepended.c,
             attr.prepended.b,
         };
         return GetByteBuffer(uniform_params);
       }},
  };

  desc.resize_function = [attr](const std::vector<BHWC>& src_shapes,
                                const std::vector<BHWC>& dst_shapes) {
    const uint3 groups_size{16, 16, 1};
    const int dst_layers = DivideRoundUp(dst_shapes[0].c, 4);
    int groups_x = DivideRoundUp(dst_shapes[0].w, groups_size.x);
    int groups_y = DivideRoundUp(dst_shapes[0].h, groups_size.y);
    int groups_z = DivideRoundUp(dst_layers, groups_size.z);
    return std::make_pair(groups_size, uint3{groups_x, groups_y, groups_z});
  };

  return desc;
}

}  // namespace metal
}  // namespace gpu
}  // namespace tflite
<commit_msg>Using Tensor objects and args in Padding kernel.<commit_after>/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/lite/delegates/gpu/metal/kernels/padding.h"

#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "absl/strings/substitute.h"
#include "tensorflow/lite/delegates/gpu/common/model.h"
#include "tensorflow/lite/delegates/gpu/common/operations.h"
#include "tensorflow/lite/delegates/gpu/common/types.h"
#include "tensorflow/lite/delegates/gpu/common/util.h"
#include "tensorflow/lite/delegates/gpu/metal/compute_task_descriptor.h"

namespace tflite {
namespace gpu {
namespace metal {
namespace {

std::string GetPaddingCode(const PadAttributes& attr) {
  const std::string channels[] = {".x", ".y", ".z", ".w"};
  std::string code = R"(
    #include <metal_stdlib>
    using namespace metal;
)";
  if (attr.type == PaddingContentType::REFLECT) {
    code += R"(
    int reflect(int x, int size) {
      return size - 1 - abs(abs(x) - size + 1);
    })";
  }
  code += R"(
    $0
    kernel void ComputeFunction(
                                $1
                                uint3 gid[[thread_position_in_grid]]) {
      if (static_cast<int>(gid.x) >= args.dst_tensor.Width() ||
          static_cast<int>(gid.y) >= args.dst_tensor.Height()) {
        return;
      }

      FLT4 value = FLT4(0.0f);
      int s_x = static_cast<int>(gid.x) - args.padding_w;
      int s_y = static_cast<int>(gid.y) - args.padding_h;)";
  if (attr.type == PaddingContentType::REFLECT) {
    code += R"(
      s_x = reflect(s_x, args.src_tensor.Width());
      s_y = reflect(s_y, args.src_tensor.Height());
)";
    if (attr.prepended.c == 0 && attr.appended.c == 0) {
      // optimized case
      code += "      value = args.src_tensor.Read(s_x, s_y, gid.z);\n";
    } else {
      code += "      int start_channel = static_cast<int>(gid.z) * 4;\n";
      for (int i = 0; i < 4; ++i) {
        const auto& s = channels[i];
        code += "      {\n";
        code += "        int channel = start_channel + " + std::to_string(i) +
                ";\n";
        code += "        int s_z = channel - args.padding_c;\n";
        // We need additional clamp for z, so that we use alignment for channels
        // and can proceed extra channels that can lead to reading out of
        // resource.
        code +=
            "        s_z = clamp(reflect(s_z, args.src_tensor.Channels()), 0, "
            "args.src_tensor.Channels() - 1);\n";
        code += "        FLT4 t = args.src_tensor.Read(s_x, s_y, s_z / 4);\n";
        code += "        FLT t_ar[4] = {t.x, t.y, t.z, t.w};\n";
        code += "        value" + s + " = t_ar[s_z % 4];\n";
        code += "      }\n";
      }
    }
  } else {
    code += R"(
      bool inside_x = s_x >= 0 && s_x < args.src_tensor.Width();
      bool inside_y = s_y >= 0 && s_y < args.src_tensor.Height();
      if (inside_x && inside_y) {
        int start_channel = static_cast<int>(gid.z) * 4;
    )";
    if (attr.prepended.c == 0 && attr.appended.c == 0) {
      // optimized case
      code += "        value = args.src_tensor.Read(s_x, s_y, gid.z);\n";
    } else if (attr.prepended.c % 4 == 0) {
      code += R"(
        int s_z = static_cast<int>(gid.z) - args.padding_c / 4;
        if (s_z >= 0 && s_z < args.src_tensor.Slices()) {
          value = args.src_tensor.Read(s_x, s_y, s_z);
        })";
    } else {
      for (int i = 0; i < 4; ++i) {
        const auto& s = channels[i];
        code += "    {\n";
        code +=
            "    int channel = start_channel + " + std::to_string(i) + ";\n";
        code += "    int s_z = channel - args.padding_c;\n";
        code += "    if (s_z >= 0 && s_z < args.src_tensor.Channels()) {\n";
        code += "      FLT4 t = args.src_tensor.Read(s_x, s_y, s_z / 4);\n";
        code += "      FLT t_ar[4] = {t.x, t.y, t.z, t.w};\n";
        code += "      value" + s + " = t_ar[s_z % 4];\n";
        code += "    }\n";
        code += "    }\n";
      }
    }
    code += "  }\n";
  }
  code += "  args.dst_tensor.GetAddress(linear_index, gid.x, gid.y, gid.z);\n";
  code += "  $2\n";
  code += "  args.dst_tensor.Write(value, gid.x, gid.y, gid.z);\n";
  code += "}\n";
  return code;
}
}  // namespace

ComputeTaskDescriptor Padding(const OperationDef& definition,
                              const PadAttributes& attr) {
  ComputeTaskDescriptor desc(definition);
  desc.tensors_as_args = true;
  desc.shader_source = GetPaddingCode(attr);

  desc.AddSrcTensor("src_tensor", definition.src_tensors[0]);
  desc.AddDstTensor("dst_tensor", definition.dst_tensors[0]);

  desc.args.AddInt("padding_w", attr.prepended.w);
  desc.args.AddInt("padding_h", attr.prepended.h);
  desc.args.AddInt("padding_c", attr.prepended.c);
  desc.args.AddInt("padding_b", attr.prepended.b);

  desc.resize_function = [attr](const std::vector<BHWC>& src_shapes,
                                const std::vector<BHWC>& dst_shapes) {
    const uint3 groups_size{16, 16, 1};
    const int dst_layers = DivideRoundUp(dst_shapes[0].c, 4);
    int groups_x = DivideRoundUp(dst_shapes[0].w, groups_size.x);
    int groups_y = DivideRoundUp(dst_shapes[0].h, groups_size.y);
    int groups_z = DivideRoundUp(dst_layers, groups_size.z);
    return std::make_pair(groups_size, uint3{groups_x, groups_y, groups_z});
  };

  return desc;
}

}  // namespace metal
}  // namespace gpu
}  // namespace tflite
<|endoftext|>