<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2016.  All rights reserved
 */
#include    "../StroikaPreComp.h"

#include    <exception>
#include    <malloc.h>

#if     qPlatform_POSIX
#include    <unistd.h>
#endif

#include    "../Memory/Common.h"

#include    "MallocGuard.h"

using   namespace   Stroika::Foundation;
using   Memory::Byte;

#if     qStroika_Foundation_Debug_MallocGuard
namespace {
    using   GuradBytes_ =   Byte[16];
    constexpr   GuradBytes_ kMallocGuardHeader_ =   { 0xf3, 0xfa, 0x0b, 0x93, 0x48, 0x50, 0x46, 0xe6, 0x22, 0xf1, 0xfa, 0xc0, 0x9a, 0x0b, 0xeb, 0x23, };
    constexpr   GuradBytes_ kMallocGuardFooter_ =   { 0x07, 0x41, 0xa4, 0x2b, 0xba, 0x97, 0xcb, 0x38, 0x46, 0x1e, 0x3c, 0x42, 0x3c, 0x5f, 0x0c, 0x80, };
    constexpr   Byte        kDeadMansLand_[]    =   { 0x1d, 0xb6, 0x20, 0x27, 0x43, 0x7a, 0x3d, 0x1a, 0x13, 0x65, };

    // need header with size, so we can ....
    struct alignas(alignof(long double))  HeaderOrFooter_ {
        GuradBytes_ fGuard;
        size_t      fRequestedBlockSize;
    };

    void    OhShit_ ()
    {
        static  bool    sDone_      { false };      // doing terminate MIGHT allocate more memory ... just go with the flow if that happens - and dont re-barf (e.g. allow backtrace if possible)
        if (not sDone_) {
            sDone_ = true;
            const char  kMsg_[] =   "Fatal Error detected in Stroika Malloc Guard\n";
            ::write (2, kMsg_, NEltsOf (kMsg_));
            std::terminate ();
        }
    }

    void*   ExposedPtrToBackendPtr_ (void* p)
    {
        if (p == nullptr) {
            OhShit_ ();
        }
        return reinterpret_cast<HeaderOrFooter_*> (p) - 1;
    }
    void*   BackendPtrToExposedPtr_ (void* p)
    {
        if (p == nullptr) {
            OhShit_ ();
        }
        return reinterpret_cast<HeaderOrFooter_*> (p) + 1;
    }
    size_t  AdjustMallocSize_ (size_t s)
    {
        return s + 2 * sizeof (HeaderOrFooter_);
    }

    bool    IsDeadMansLand_ (const Byte* s, const Byte* e)
    {
        return false;
    }
    void    SetDeadMansLand_ (Byte* s, Byte* e)
    {
        const   Byte*   pBadFillStart   =   begin (kDeadMansLand_);
        const   Byte*   pBadFillEnd     =   end (kDeadMansLand_);
        const   Byte*   badFillI        =   pBadFillStart;
        for (Byte* oi = s; oi != e; ++oi) {
            *oi = *badFillI;
            badFillI++;
            if (badFillI == pBadFillEnd) {
                badFillI = pBadFillStart;
            }
        }
    }
    void    SetDeadMansLand_ (void* p)
    {
        const HeaderOrFooter_*  hp  =   reinterpret_cast<const HeaderOrFooter_*> (p);
        SetDeadMansLand_ (reinterpert_cast<Byte*> (p), reinterpert_cast<Byte*> (p) + AdjustMallocSize_ (hp->fRequestedBlockSize));
    }

    /*
     *  not 100% threadsafe, but OK.
     *
     *  Also FreeList alway uses BackendPtr - not ExternalPtr
     */
    void*   sFreeList_[100];
    void**  sFreeList_NextFreeI_    =   &sFreeList_[0];
    void    Add2FreeList_ (void* p)
    {
        *sFreeList_NextFreeI_ = p;
        void**  next = sFreeList_NextFreeI_ + 1;
        if (next >= end (sFreeList_)) {
            next = begin (sFreeList_);
        }
        sFreeList_NextFreeI_ = next;    // race in that we could SKIP recording a free element, but thats harmless - just a missed opportunity to detect an error
    }
    void    ClearFromFreeList_ (void* p)
    {
        // not a race because you cannot free and allocate the same pointer at the same time
        for (void** i = begin (sFreeList_); i != end (sFreeList_); ++i) {
            if (*i == p) {
                *i = nullptr;
            }
        }
    }
    bool    IsInFreeList_ (void* p)
    {
        for (void** i = begin (sFreeList_); i != end (sFreeList_); ++i) {
            if (*i == p) {
                return true;
            }
        }
        return false;
    }

    void    Validate_ (const HeaderOrFooter_& header, const HeaderOrFooter_& footer)
    {
        if (::memcmp (&header.fGuard, &kMallocGuardHeader_, sizeof (kMallocGuardHeader_)) != 0) {
            OhShit_ ();
        }
        if (::memcmp (&footer.fGuard, &kMallocGuardFooter_, sizeof (kMallocGuardFooter_)) != 0) {
            OhShit_ ();
        }
        if (header.fRequestedBlockSize != footer.fRequestedBlockSize) {
            OhShit_ ();
        }
        // OK
    }
    void    ValidateBackendPtr_ (const void* p)
    {
        const HeaderOrFooter_*  hp  =   reinterpret_cast<const HeaderOrFooter_*> (p);
        const HeaderOrFooter_*  fp  =   reinterpret_cast<const HeaderOrFooter_*> (reinterpret_cast<const Byte*> (hp + 1) + hp->fRequestedBlockSize);
        HeaderOrFooter_ footer;
        (void)::memcpy (&footer, fp, sizeof (footer));  // align access
        Validate_ (*hp, footer);
        if (IsInFreeList_ (p)) {
            OhShit_ ();
        }
    }
    void   PatchNewPointer_ (void* p, size_t requestedSize)
    {
        HeaderOrFooter_*  hp   =   reinterpret_cast< HeaderOrFooter_*> (p);
        (void)::memcpy (begin (hp->fGuard), begin (kMallocGuardHeader_),  NEltsOf (kMallocGuardHeader_));
        hp->fRequestedBlockSize = requestedSize;
        HeaderOrFooter_*  fp  =    reinterpret_cast< HeaderOrFooter_*> (reinterpret_cast<Byte*> (hp + 1) + hp->fRequestedBlockSize);
        (void)::memcpy (begin (fp->fGuard), begin (kMallocGuardFooter_),  NEltsOf (kMallocGuardFooter_));
        fp->fRequestedBlockSize = requestedSize;
    }
}
#endif

#if     qStroika_Foundation_Debug_MallocGuard

extern "C"  void    __libc_free (void* __ptr);
extern "C"  void*   __libc_malloc (size_t __size);
extern "C"  void*   __libc_realloc (void* __ptr, size_t __size);
extern "C"  void*   __libc_calloc (size_t __nmemb, size_t __size);
extern "C"  void    __libc_free (void* __ptr);
extern "C"  void*   __libc_memalign (size_t __alignment, size_t __size);

#if 0
weak_alias (__malloc_info, malloc_info)
5256
5257
5258    strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
5259    strong_alias (__libc_free, __cfree) weak_alias (__libc_free, cfree)
5260    strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
5261    strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
5262    strong_alias (__libc_memalign, __memalign)
5263    weak_alias (__libc_memalign, memalign)
5264    strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
5265    strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
5266    strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
5267    strong_alias (__libc_mallinfo, __mallinfo)
5268    weak_alias (__libc_mallinfo, mallinfo)
5269    strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
5270
5271    weak_alias (__malloc_stats, malloc_stats)
5272    weak_alias (__malloc_usable_size, malloc_usable_size)
5273    weak_alias (__malloc_trim, malloc_trim)
5274    weak_alias (__malloc_get_state, malloc_get_state)
5275    weak_alias (__malloc_set_state, malloc_set_state)
#endif

extern "C"  void*   calloc (size_t __nmemb, size_t __size)
{
    size_t  n   =   __nmemb * __size;
    void*   p   =   malloc (n);
    (void)::memset (p, 0, n);
    return p;
}

extern "C"  void    cfree (void* __ptr)
{
    free (__ptr);
}

extern "C"  void    free (void* __ptr)
{
    if (__ptr == nullptr) {
        // according to http://linux.die.net/man/3/free
        // "if ptr is NULL, no operation is performed." - and glibc does call this internally
        return;
    }
    void*   p = ExposedPtrToBackendPtr_ (__ptr);
    ValidateBackendPtr_ (p);
    SetDeadMansLand_ (p);
    Add2FreeList_ (p);
    __libc_free (p);
}

extern "C"  void*   malloc (size_t __size)
{
    void*   p   =   __libc_malloc (AdjustMallocSize_ (__size));
    PatchNewPointer_ (p, __size);
    ClearFromFreeList_ (p);
    ValidateBackendPtr_ (p);
    if (p != nullptr) {
        p = BackendPtrToExposedPtr_ (p);
    }
    return p;
}

extern "C"  void*    realloc (void* __ptr, size_t __size)
{
    if (__ptr == nullptr) {
        // from http://linux.die.net/man/3/realloc
        // If ptr is NULL, then the call is equivalent to malloc(size),
        return malloc (__size);
    }
    if (__ptr != nullptr and __size == 0) {
        // from http://linux.die.net/man/3/realloc
        // if size is equal to zero, and ptr is not NULL, then the call is equivalent to free(ptr).
        free (__ptr);
        return nullptr;
    }
    void*   p   =   ExposedPtrToBackendPtr_ (__ptr);
    ValidateBackendPtr_ (p);
    size_t  n   =   AdjustMallocSize_ (__size);
    void*   newP = __libc_realloc (p, n);
    if (newP != nullptr) {
        PatchNewPointer_ (newP, __size);
        if (newP != p) {
            SetDeadMansLand_ (p);
            Add2FreeList_ (p);
            ClearFromFreeList_ (newP);
        }
        ValidateBackendPtr_ (newP);
        newP = BackendPtrToExposedPtr_ (newP);
    }
    return newP;
}

extern "C"  void*   valloc (size_t __size)
{
    OhShit_ ();
    return nullptr;
}

extern "C"  void*   pvalloc (size_t __size)
{
    OhShit_ ();
    return nullptr;
}

extern "C"  void*   memalign (size_t __alignment, size_t __size)
{
    OhShit_ ();
    return nullptr;
}

extern "C"  size_t malloc_usable_size (void* ptr)
{
    if (ptr == nullptr) {
        return 0;
    }
    void*   p   =   ExposedPtrToBackendPtr_ (ptr);
    ValidateBackendPtr_ (p);
    const HeaderOrFooter_*  hp  =   reinterpret_cast<const HeaderOrFooter_*> (p);
    return hp->fRequestedBlockSize;
}
#endif
<commit_msg>fixed typo in MallocGuard code<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2016.  All rights reserved
 */
#include    "../StroikaPreComp.h"

#include    <exception>
#include    <malloc.h>

#if     qPlatform_POSIX
#include    <unistd.h>
#endif

#include    "../Memory/Common.h"

#include    "MallocGuard.h"

using   namespace   Stroika::Foundation;
using   Memory::Byte;

#if     qStroika_Foundation_Debug_MallocGuard
namespace {
    using   GuradBytes_ =   Byte[16];
    constexpr   GuradBytes_ kMallocGuardHeader_ =   { 0xf3, 0xfa, 0x0b, 0x93, 0x48, 0x50, 0x46, 0xe6, 0x22, 0xf1, 0xfa, 0xc0, 0x9a, 0x0b, 0xeb, 0x23, };
    constexpr   GuradBytes_ kMallocGuardFooter_ =   { 0x07, 0x41, 0xa4, 0x2b, 0xba, 0x97, 0xcb, 0x38, 0x46, 0x1e, 0x3c, 0x42, 0x3c, 0x5f, 0x0c, 0x80, };
    constexpr   Byte        kDeadMansLand_[]    =   { 0x1d, 0xb6, 0x20, 0x27, 0x43, 0x7a, 0x3d, 0x1a, 0x13, 0x65, };

    // need header with size, so we can ....
    struct alignas(alignof(long double))  HeaderOrFooter_ {
        GuradBytes_ fGuard;
        size_t      fRequestedBlockSize;
    };

    void    OhShit_ ()
    {
        static  bool    sDone_      { false };      // doing terminate MIGHT allocate more memory ... just go with the flow if that happens - and dont re-barf (e.g. allow backtrace if possible)
        if (not sDone_) {
            sDone_ = true;
            const char  kMsg_[] =   "Fatal Error detected in Stroika Malloc Guard\n";
            ::write (2, kMsg_, NEltsOf (kMsg_));
            std::terminate ();
        }
    }

    void*   ExposedPtrToBackendPtr_ (void* p)
    {
        if (p == nullptr) {
            OhShit_ ();
        }
        return reinterpret_cast<HeaderOrFooter_*> (p) - 1;
    }
    void*   BackendPtrToExposedPtr_ (void* p)
    {
        if (p == nullptr) {
            OhShit_ ();
        }
        return reinterpret_cast<HeaderOrFooter_*> (p) + 1;
    }
    size_t  AdjustMallocSize_ (size_t s)
    {
        return s + 2 * sizeof (HeaderOrFooter_);
    }

    bool    IsDeadMansLand_ (const Byte* s, const Byte* e)
    {
        return false;
    }
    void    SetDeadMansLand_ (Byte* s, Byte* e)
    {
        const   Byte*   pBadFillStart   =   begin (kDeadMansLand_);
        const   Byte*   pBadFillEnd     =   end (kDeadMansLand_);
        const   Byte*   badFillI        =   pBadFillStart;
        for (Byte* oi = s; oi != e; ++oi) {
            *oi = *badFillI;
            badFillI++;
            if (badFillI == pBadFillEnd) {
                badFillI = pBadFillStart;
            }
        }
    }
    void    SetDeadMansLand_ (void* p)
    {
        const HeaderOrFooter_*  hp  =   reinterpret_cast<const HeaderOrFooter_*> (p);
        SetDeadMansLand_ (reinterpret_cast<Byte*> (p), reinterpret_cast<Byte*> (p) + AdjustMallocSize_ (hp->fRequestedBlockSize));
    }

    /*
     *  not 100% threadsafe, but OK.
     *
     *  Also FreeList alway uses BackendPtr - not ExternalPtr
     */
    void*   sFreeList_[100];
    void**  sFreeList_NextFreeI_    =   &sFreeList_[0];
    void    Add2FreeList_ (void* p)
    {
        *sFreeList_NextFreeI_ = p;
        void**  next = sFreeList_NextFreeI_ + 1;
        if (next >= end (sFreeList_)) {
            next = begin (sFreeList_);
        }
        sFreeList_NextFreeI_ = next;    // race in that we could SKIP recording a free element, but thats harmless - just a missed opportunity to detect an error
    }
    void    ClearFromFreeList_ (void* p)
    {
        // not a race because you cannot free and allocate the same pointer at the same time
        for (void** i = begin (sFreeList_); i != end (sFreeList_); ++i) {
            if (*i == p) {
                *i = nullptr;
            }
        }
    }
    bool    IsInFreeList_ (const void* p)
    {
        for (void** i = begin (sFreeList_); i != end (sFreeList_); ++i) {
            if (*i == p) {
                return true;
            }
        }
        return false;
    }

    void    Validate_ (const HeaderOrFooter_& header, const HeaderOrFooter_& footer)
    {
        if (::memcmp (&header.fGuard, &kMallocGuardHeader_, sizeof (kMallocGuardHeader_)) != 0) {
            OhShit_ ();
        }
        if (::memcmp (&footer.fGuard, &kMallocGuardFooter_, sizeof (kMallocGuardFooter_)) != 0) {
            OhShit_ ();
        }
        if (header.fRequestedBlockSize != footer.fRequestedBlockSize) {
            OhShit_ ();
        }
        // OK
    }
    void    ValidateBackendPtr_ (const void* p)
    {
        const HeaderOrFooter_*  hp  =   reinterpret_cast<const HeaderOrFooter_*> (p);
        const HeaderOrFooter_*  fp  =   reinterpret_cast<const HeaderOrFooter_*> (reinterpret_cast<const Byte*> (hp + 1) + hp->fRequestedBlockSize);
        HeaderOrFooter_ footer;
        (void)::memcpy (&footer, fp, sizeof (footer));  // align access
        Validate_ (*hp, footer);
        if (IsInFreeList_ (p)) {
            OhShit_ ();
        }
    }
    void   PatchNewPointer_ (void* p, size_t requestedSize)
    {
        HeaderOrFooter_*  hp   =   reinterpret_cast< HeaderOrFooter_*> (p);
        (void)::memcpy (begin (hp->fGuard), begin (kMallocGuardHeader_),  NEltsOf (kMallocGuardHeader_));
        hp->fRequestedBlockSize = requestedSize;
        HeaderOrFooter_*  fp  =    reinterpret_cast< HeaderOrFooter_*> (reinterpret_cast<Byte*> (hp + 1) + hp->fRequestedBlockSize);
        (void)::memcpy (begin (fp->fGuard), begin (kMallocGuardFooter_),  NEltsOf (kMallocGuardFooter_));
        fp->fRequestedBlockSize = requestedSize;
    }
}
#endif

#if     qStroika_Foundation_Debug_MallocGuard

extern "C"  void    __libc_free (void* __ptr);
extern "C"  void*   __libc_malloc (size_t __size);
extern "C"  void*   __libc_realloc (void* __ptr, size_t __size);
extern "C"  void*   __libc_calloc (size_t __nmemb, size_t __size);
extern "C"  void    __libc_free (void* __ptr);
extern "C"  void*   __libc_memalign (size_t __alignment, size_t __size);

#if 0
weak_alias (__malloc_info, malloc_info)
5256
5257
5258    strong_alias (__libc_calloc, __calloc) weak_alias (__libc_calloc, calloc)
5259    strong_alias (__libc_free, __cfree) weak_alias (__libc_free, cfree)
5260    strong_alias (__libc_free, __free) strong_alias (__libc_free, free)
5261    strong_alias (__libc_malloc, __malloc) strong_alias (__libc_malloc, malloc)
5262    strong_alias (__libc_memalign, __memalign)
5263    weak_alias (__libc_memalign, memalign)
5264    strong_alias (__libc_realloc, __realloc) strong_alias (__libc_realloc, realloc)
5265    strong_alias (__libc_valloc, __valloc) weak_alias (__libc_valloc, valloc)
5266    strong_alias (__libc_pvalloc, __pvalloc) weak_alias (__libc_pvalloc, pvalloc)
5267    strong_alias (__libc_mallinfo, __mallinfo)
5268    weak_alias (__libc_mallinfo, mallinfo)
5269    strong_alias (__libc_mallopt, __mallopt) weak_alias (__libc_mallopt, mallopt)
5270
5271    weak_alias (__malloc_stats, malloc_stats)
5272    weak_alias (__malloc_usable_size, malloc_usable_size)
5273    weak_alias (__malloc_trim, malloc_trim)
5274    weak_alias (__malloc_get_state, malloc_get_state)
5275    weak_alias (__malloc_set_state, malloc_set_state)
#endif

extern "C"  void*   calloc (size_t __nmemb, size_t __size)
{
    size_t  n   =   __nmemb * __size;
    void*   p   =   malloc (n);
    (void)::memset (p, 0, n);
    return p;
}

extern "C"  void    cfree (void* __ptr)
{
    free (__ptr);
}

extern "C"  void    free (void* __ptr)
{
    if (__ptr == nullptr) {
        // according to http://linux.die.net/man/3/free
        // "if ptr is NULL, no operation is performed." - and glibc does call this internally
        return;
    }
    void*   p = ExposedPtrToBackendPtr_ (__ptr);
    ValidateBackendPtr_ (p);
    SetDeadMansLand_ (p);
    Add2FreeList_ (p);
    __libc_free (p);
}

extern "C"  void*   malloc (size_t __size)
{
    void*   p   =   __libc_malloc (AdjustMallocSize_ (__size));
    PatchNewPointer_ (p, __size);
    ClearFromFreeList_ (p);
    ValidateBackendPtr_ (p);
    if (p != nullptr) {
        p = BackendPtrToExposedPtr_ (p);
    }
    return p;
}

extern "C"  void*    realloc (void* __ptr, size_t __size)
{
    if (__ptr == nullptr) {
        // from http://linux.die.net/man/3/realloc
        // If ptr is NULL, then the call is equivalent to malloc(size),
        return malloc (__size);
    }
    if (__ptr != nullptr and __size == 0) {
        // from http://linux.die.net/man/3/realloc
        // if size is equal to zero, and ptr is not NULL, then the call is equivalent to free(ptr).
        free (__ptr);
        return nullptr;
    }
    void*   p   =   ExposedPtrToBackendPtr_ (__ptr);
    ValidateBackendPtr_ (p);
    size_t  n   =   AdjustMallocSize_ (__size);
    void*   newP = __libc_realloc (p, n);
    if (newP != nullptr) {
        PatchNewPointer_ (newP, __size);
        if (newP != p) {
            SetDeadMansLand_ (p);
            Add2FreeList_ (p);
            ClearFromFreeList_ (newP);
        }
        ValidateBackendPtr_ (newP);
        newP = BackendPtrToExposedPtr_ (newP);
    }
    return newP;
}

extern "C"  void*   valloc (size_t __size)
{
    OhShit_ ();
    return nullptr;
}

extern "C"  void*   pvalloc (size_t __size)
{
    OhShit_ ();
    return nullptr;
}

extern "C"  void*   memalign (size_t __alignment, size_t __size)
{
    OhShit_ ();
    return nullptr;
}

extern "C"  size_t malloc_usable_size (void* ptr)
{
    if (ptr == nullptr) {
        return 0;
    }
    void*   p   =   ExposedPtrToBackendPtr_ (ptr);
    ValidateBackendPtr_ (p);
    const HeaderOrFooter_*  hp  =   reinterpret_cast<const HeaderOrFooter_*> (p);
    return hp->fRequestedBlockSize;
}
#endif
<|endoftext|>


<commit_before>/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 *  ola-discover.cpp
 *  Print the list of UIDs and force RDM discovery
 *  Copyright (C) 2010 Simon Newton
 */

#include <errno.h>
#include <getopt.h>
#include <ola/Logging.h>
#include <ola/OlaCallbackClient.h>
#include <ola/OlaClientWrapper.h>
#include <ola/network/SelectServer.h>

#include <iostream>
#include <iomanip>
#include <string>
#include <vector>

using std::cout;
using std::cerr;
using std::endl;
using std::setw;
using std::string;
using std::vector;
using ola::OlaCallbackClient;
using ola::OlaCallbackClientWrapper;
using ola::network::SelectServer;
using ola::rdm::UIDSet;

static const int INVALID_VALUE = -1;

typedef struct {
  int uni;         // universe id
  bool help;       // show the help
  bool full;       // full discovery
  bool incremental; // incremental discovery
  string cmd;      // argv[0]
} options;

SelectServer *ss;

/*
 * This is called when we recieve uids for a universe
 * @param universes a vector of OlaUniverses
 */
void UIDList(const ola::rdm::UIDSet &uids,
                       const string &error) {
  if (error.empty()) {
    UIDSet::Iterator iter = uids.Begin();
    for (; iter != uids.End(); ++iter) {
      cout << *iter << endl;
    }
  } else {
    cout << error << endl;
  }
  ss->Terminate();
}

/*
 * Called once we get an ack for the discovery request
 */
void ForceRDMDiscoveryComplete(const string &error) {
  if (!error.empty())
    cout << error << endl;
  ss->Terminate();
}

/*
 * parse our cmd line options
 */
void ParseOptions(int argc, char *argv[], options *opts) {
  opts->uni = INVALID_VALUE;
  opts->help = false;
  opts->full = false;
  opts->incremental = false;

  static struct option long_options[] = {
      {"help", no_argument, 0, 'h'},
      {"full", no_argument, 0, 'f'},
      {"incremental", no_argument, 0, 'i'},
      {"universe", required_argument, 0, 'u'},
      {0, 0, 0, 0}
    };

  int c;
  int option_index = 0;

  while (1) {
    c = getopt_long(argc, argv, "u:fhi", long_options, &option_index);

    if (c == -1)
      break;

    switch (c) {
      case 0:
        break;
      case 'f':
        opts->full = true;
        break;
      case 'h':
        opts->help = true;
        break;
      case 'i':
        opts->incremental = true;
        break;
      case 'u':
        opts->uni = atoi(optarg);
        break;
      case '?':
        break;
      default:
        break;
    }
  }
}

/*
 * Help message for set dmx
 */
void DisplayGetUIDsHelp(const options &opts) {
  cout << "Usage: " << opts.cmd <<
  " --universe <universe> [--force_discovery]\n"
  "\n"
  "Fetch the UID list for a universe.\n"
  "\n"
  "  -h, --help        Display this help message and exit.\n"
  "  -f, --full        Force full RDM Discovery for this universe\n"
  "  -i, --incremental Force incremental RDM Discovery for this universe\n"
  "  -u, --universe <universe> Universe number.\n"
  << endl;
}

/*
 * Send a fetch uid list request
 * @param client  the ola client
 * @param opts  the const options
 */
bool FetchUIDs(OlaCallbackClient *client, const options &opts) {
  if (opts.uni == INVALID_VALUE) {
    DisplayGetUIDsHelp(opts);
    return false;
  }

  if (opts.full) {
    return client->RunDiscovery(
        opts.uni,
        true,
        ola::NewSingleCallback(&ForceRDMDiscoveryComplete));
  } else if (opts.incremental) {
    return client->RunDiscovery(
        opts.uni,
        false,
        ola::NewSingleCallback(&ForceRDMDiscoveryComplete));
  } else {
    return client->FetchUIDList(
        opts.uni,
        ola::NewSingleCallback(&UIDList));
  }
}

/*
 * Main
 */
int main(int argc, char *argv[]) {
  ola::InitLogging(ola::OLA_LOG_WARN, ola::OLA_LOG_STDERR);
  OlaCallbackClientWrapper ola_client;
  options opts;
  opts.cmd = argv[0];

  ParseOptions(argc, argv, &opts);

  if (opts.help)
    DisplayGetUIDsHelp(opts);

  if (opts.full && opts.incremental) {
    cerr << "Only one of -i and -f can be specified" << endl;
    exit(1);
  }

  if (!ola_client.Setup()) {
    OLA_FATAL << "Setup failed";
    exit(1);
  }

  OlaCallbackClient *client = ola_client.GetClient();
  ss = ola_client.GetSelectServer();

  if (FetchUIDs(client, opts))
    ss->Run();
  return 0;
}
<commit_msg> * examples: fix the help flag<commit_after>/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU Library General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 *  ola-discover.cpp
 *  Print the list of UIDs and force RDM discovery
 *  Copyright (C) 2010 Simon Newton
 */

#include <errno.h>
#include <getopt.h>
#include <ola/Logging.h>
#include <ola/OlaCallbackClient.h>
#include <ola/OlaClientWrapper.h>
#include <sysexits.h>
#include <ola/network/SelectServer.h>

#include <iostream>
#include <iomanip>
#include <string>
#include <vector>

using std::cout;
using std::cerr;
using std::endl;
using std::setw;
using std::string;
using std::vector;
using ola::OlaCallbackClient;
using ola::OlaCallbackClientWrapper;
using ola::network::SelectServer;
using ola::rdm::UIDSet;

static const int INVALID_VALUE = -1;

typedef struct {
  int uni;         // universe id
  bool help;       // show the help
  bool full;       // full discovery
  bool incremental;  // incremental discovery
  string cmd;      // argv[0]
} options;

SelectServer *ss;

/*
 * This is called when we recieve uids for a universe
 * @param universes a vector of OlaUniverses
 */
void UIDList(const ola::rdm::UIDSet &uids,
                       const string &error) {
  if (error.empty()) {
    UIDSet::Iterator iter = uids.Begin();
    for (; iter != uids.End(); ++iter) {
      cout << *iter << endl;
    }
  } else {
    cout << error << endl;
  }
  ss->Terminate();
}

/*
 * Called once we get an ack for the discovery request
 */
void ForceRDMDiscoveryComplete(const string &error) {
  if (!error.empty())
    cout << error << endl;
  ss->Terminate();
}

/*
 * parse our cmd line options
 */
void ParseOptions(int argc, char *argv[], options *opts) {
  opts->uni = INVALID_VALUE;
  opts->help = false;
  opts->full = false;
  opts->incremental = false;

  static struct option long_options[] = {
      {"help", no_argument, 0, 'h'},
      {"full", no_argument, 0, 'f'},
      {"incremental", no_argument, 0, 'i'},
      {"universe", required_argument, 0, 'u'},
      {0, 0, 0, 0}
    };

  int c;
  int option_index = 0;

  while (1) {
    c = getopt_long(argc, argv, "u:fhi", long_options, &option_index);

    if (c == -1)
      break;

    switch (c) {
      case 0:
        break;
      case 'f':
        opts->full = true;
        break;
      case 'h':
        opts->help = true;
        break;
      case 'i':
        opts->incremental = true;
        break;
      case 'u':
        opts->uni = atoi(optarg);
        break;
      case '?':
        break;
      default:
        break;
    }
  }
}

/*
 * Help message for set dmx
 */
void DisplayGetUIDsHelp(const options &opts) {
  cout << "Usage: " << opts.cmd <<
  " --universe <universe> [--force_discovery]\n"
  "\n"
  "Fetch the UID list for a universe.\n"
  "\n"
  "  -h, --help        Display this help message and exit.\n"
  "  -f, --full        Force full RDM Discovery for this universe\n"
  "  -i, --incremental Force incremental RDM Discovery for this universe\n"
  "  -u, --universe <universe> Universe number.\n"
  << endl;
}

/*
 * Send a fetch uid list request
 * @param client  the ola client
 * @param opts  the const options
 */
bool FetchUIDs(OlaCallbackClient *client, const options &opts) {
  if (opts.uni == INVALID_VALUE) {
    DisplayGetUIDsHelp(opts);
    return false;
  }

  if (opts.full) {
    return client->RunDiscovery(
        opts.uni,
        true,
        ola::NewSingleCallback(&ForceRDMDiscoveryComplete));
  } else if (opts.incremental) {
    return client->RunDiscovery(
        opts.uni,
        false,
        ola::NewSingleCallback(&ForceRDMDiscoveryComplete));
  } else {
    return client->FetchUIDList(
        opts.uni,
        ola::NewSingleCallback(&UIDList));
  }
}

/*
 * Main
 */
int main(int argc, char *argv[]) {
  ola::InitLogging(ola::OLA_LOG_WARN, ola::OLA_LOG_STDERR);
  OlaCallbackClientWrapper ola_client;
  options opts;
  opts.cmd = argv[0];

  ParseOptions(argc, argv, &opts);

  if (opts.help) {
    DisplayGetUIDsHelp(opts);
    exit(EX_OK);
  }

  if (opts.full && opts.incremental) {
    cerr << "Only one of -i and -f can be specified" << endl;
    exit(EX_USAGE);
  }

  if (!ola_client.Setup()) {
    OLA_FATAL << "Setup failed";
    exit(EX_UNAVAILABLE);
  }

  OlaCallbackClient *client = ola_client.GetClient();
  ss = ola_client.GetSelectServer();

  if (FetchUIDs(client, opts))
    ss->Run();
  return EX_OK;
}
<|endoftext|>


<commit_before>//==========================================================================================
// Before using the macro makeWeights.C you should already have available the output .root
// files from various methods from the previous run over data (without any weights). When
// calling this macro you must specify the analysis type and the method from which output
// file you would like to make the weights for the next run (for the cumulants, GFC and QC,
// you must also specify the order):
//
// 1. type of analysis can be: ESD, AOD, MC, ESDMC0 or ESDMC1;
//
// 2. method can be: MCEP, LYZ1, LYZ2, LYZEP, FQD, GFC or QC;
//
// 3. cumulant order can be: 2nd, 4th, 6th or 8th.
//==========================================================================================

void makeWeights(TString type="ESD", TString method="GFC", TString cumulantOrder="4th")
{
 //load needed libraries:
 gSystem->AddIncludePath("-I$ROOTSYS/include");
 gSystem->Load("libTree.so");

 //for AliRoot
 gSystem->AddIncludePath("-I$ALICE_ROOT/include");
 gSystem->Load("libANALYSIS.so");
 gSystem->Load("libPWG2flowCommon.so");
 cerr<<"libPWG2flowCommon.so loaded ..."<<endl;

 //open the output file from the first run of the specified method:
 TString inputFileName = "output";
 TFile* file = NULL;
 file = TFile::Open(((((inputFileName.Append(method.Data())).Append("analysis")).Append(type.Data())).Append(".root")).Data(), "READ");

 //accessing the results:
 TString cobj = "cobj";
 TString afc  = "AliFlowCommonHist";
 TString afcr = "AliFlowCommonHistResults";

 TList *pList = NULL;
 AliFlowCommonHist *commonHist = NULL;
 AliFlowCommonHistResults *commonHistRes = NULL;

 if(file)
 {
  file->GetObject((cobj.Append(method.Data()).Data()),pList);
  if(pList)
  {
   if(!(method=="GFC"||method=="QC"))
   {
    commonHist    = dynamic_cast<AliFlowCommonHist*> (pList->FindObject((afc.Append(method.Data())).Data()));
    commonHistRes = dynamic_cast<AliFlowCommonHistResults*> (pList->FindObject((afcr.Append(method.Data())).Data()));
   }else if(method=="GFC")
    {
     commonHist    = dynamic_cast<AliFlowCommonHist*> (pList->FindObject((afc.Append(method.Data())).Data()));
     commonHistRes = dynamic_cast<AliFlowCommonHistResults*> (pList->FindObject((((afcr.Append(cumulantOrder.Data()).Append("Order")).Append(method.Data())).Data())));
    }else
     {
      commonHist    = dynamic_cast<AliFlowCommonHist*> (pList->FindObject(((afc.Append(cumulantOrder.Data())).Append("Order")).Append(method.Data())));
      commonHistRes = dynamic_cast<AliFlowCommonHistResults*> (pList->FindObject((((afcr.Append(cumulantOrder.Data()).Append("Order")).Append(method.Data())).Data())));
     }
  }//end of if(pList)
 }//end of if(file)

 //making the output file and creating the TList to hold the histograms with weights:
 TFile* outputFile = new TFile("weights.root","RECREATE");
 TList* listWeights = new TList();
 Int_t nBinsPhi = 0;
 Double_t nParticlesInBin = 0;  // number of particles in particular phi bin
 Double_t nParticlesPerBin = 0; // average number of particles per phi bin
 Double_t nParticles = 0;       // number of particles in all phi bins
 Double_t wPhi = 0.;
 //common control histos:
 if(commonHist)
 {
  //azimuthal acceptance:
  (commonHist->GetHistPhiInt())->SetName("phi_weights");
  (commonHist->GetHistPhiInt())->SetTitle("Phi_weights: correction for non-uniform acceptance");
  (commonHist->GetHistPhiInt())->SetYTitle("weights");
  (commonHist->GetHistPhiInt())->SetXTitle("#phi");
  nBinsPhi = (commonHist->GetHistPhiInt())->GetNbinsX();
  nParticles = (commonHist->GetHistPhiInt())->Integral();
  if(nBinsPhi) nParticlesPerBin = nParticles/nBinsPhi;
  // loop over phi bins:
  for(Int_t b=1;b<nBinsPhi+1;b++)
  {
   nParticlesInBin = (commonHist->GetHistPhiInt())->GetBinContent(b);
   // calculate the phi weight wPhi for each bin:
   if(nParticlesInBin) wPhi = nParticlesPerBin/nParticlesInBin;
   (commonHist->GetHistPhiInt())->SetBinContent(b,wPhi);
  }
  listWeights->Add(commonHist->GetHistPhiInt());
 }else{cout<<" WARNING: the common control histos from the 1st run were not accessed."<<endl;}
 //common results histos:
 if(commonHistRes)
 {
  //diff. flow (pt):
  (commonHistRes->GetHistDiffFlowPtPOI())->SetName("pt_weights");
  listWeights->Add(commonHistRes->GetHistDiffFlowPtPOI());
  //diff. flow (eta):
  (commonHistRes->GetHistDiffFlowEtaPOI())->SetName("eta_weights");
  listWeights->Add(commonHistRes->GetHistDiffFlowEtaPOI());
 }else{cout<<" WARNING: the common results histos from the 1st run were not accessed."<<endl;}

 outputFile->WriteObject(listWeights,"weights","SingleKey");

 delete listWeights;
 delete outputFile;
}

<commit_msg>added the option to use functions as weights<commit_after>//==========================================================================================
// Before using the macro makeWeights.C you should already have available the output .root
// files from various methods from the previous run over data (without any weights). When
// calling this macro you must specify the analysis type and the method from which output
// file you would like to make the weights for the next run (for the cumulants, GFC and QC,
// you must also specify the order):
//
// 1. type of analysis can be: ESD, AOD, MC, ESDMC0 or ESDMC1;
//
// 2. method can be: MCEP, LYZ1, LYZ2, LYZEP, FQD, GFC or QC;
//
// 3. cumulant order can be: 2nd, 4th, 6th or 8th.
//==========================================================================================

void makeWeights(TString type="ESD", TString method="GFC", TString cumulantOrder="4th")
{
 // load needed libraries:
 gSystem->AddIncludePath("-I$ROOTSYS/include");
 gSystem->Load("libTree.so");

 // for AliRoot
 gSystem->AddIncludePath("-I$ALICE_ROOT/include");
 gSystem->Load("libANALYSIS.so");
 gSystem->Load("libPWG2flowCommon.so");
 cerr<<"libPWG2flowCommon.so loaded ..."<<endl;

 // open the output file from the first run of the specified method:
 TString inputFileName = "output";
 TFile* file = NULL;
 file = TFile::Open(((((inputFileName.Append(method.Data())).Append("analysis")).Append(type.Data())).Append(".root")).Data(), "READ");

 // using pt weights linear or quadratic in pt:
 Bool_t useLinearPt    = kFALSE;
 Bool_t useQuadraticPt = kTRUE;
 if(useLinearPt && useQuadraticPt)
 {
  cout<<" WARNING: you can use pt weights linear or quadratic in pt, but not at the same time. "<<endl;
  exit(0);
 }
 Bool_t useFunctionOfPt = useLinearPt||useQuadraticPt;

 // accessing the results:
 TString cobj = "cobj";
 TString afc  = "AliFlowCommonHist";
 TString afcr = "AliFlowCommonHistResults";

 TList *pList = NULL;
 AliFlowCommonHist *commonHist = NULL;
 AliFlowCommonHistResults *commonHistRes = NULL;

 if(file)
 {
  file->GetObject((cobj.Append(method.Data()).Data()),pList);
  if(pList)
  {
   if(!(method=="GFC"||method=="QC"))
   {
    commonHist    = dynamic_cast<AliFlowCommonHist*> (pList->FindObject((afc.Append(method.Data())).Data()));
    commonHistRes = dynamic_cast<AliFlowCommonHistResults*> (pList->FindObject((afcr.Append(method.Data())).Data()));
   }else if(method=="GFC")
    {
     commonHist    = dynamic_cast<AliFlowCommonHist*> (pList->FindObject((afc.Append(method.Data())).Data()));
     commonHistRes = dynamic_cast<AliFlowCommonHistResults*> (pList->FindObject((((afcr.Append(cumulantOrder.Data()).Append("Order")).Append(method.Data())).Data())));
    }else
     {
      commonHist    = dynamic_cast<AliFlowCommonHist*> (pList->FindObject(((afc.Append(cumulantOrder.Data())).Append("Order")).Append(method.Data())));
      commonHistRes = dynamic_cast<AliFlowCommonHistResults*> (pList->FindObject((((afcr.Append(cumulantOrder.Data()).Append("Order")).Append(method.Data())).Data())));
     }
  }//end of if(pList)
 }//end of if(file)

 //making the output file and creating the TList to hold the histograms with weights:
 TFile* outputFile = new TFile("weights.root","RECREATE");
 TList* listWeights = new TList();
 Int_t nBinsPhi = 0;
 Double_t nParticlesInBin = 0;  // number of particles in particular phi bin
 Double_t nParticlesPerBin = 0; // average number of particles per phi bin
 Double_t nParticles = 0;       // number of particles in all phi bins
 Double_t wPhi = 0.;

 // common control histos:
 if(commonHist)
 {
  // azimuthal acceptance:
  (commonHist->GetHistPhiInt())->SetName("phi_weights");
  (commonHist->GetHistPhiInt())->SetTitle("phi_weights: correction for non-uniform acceptance");
  (commonHist->GetHistPhiInt())->SetYTitle("weights");
  (commonHist->GetHistPhiInt())->SetXTitle("#phi");
  nBinsPhi = (commonHist->GetHistPhiInt())->GetNbinsX();
  nParticles = (commonHist->GetHistPhiInt())->Integral();
  if(nBinsPhi) nParticlesPerBin = nParticles/nBinsPhi;
  // loop over phi bins:
  for(Int_t b=1;b<nBinsPhi+1;b++)
  {
   nParticlesInBin = (commonHist->GetHistPhiInt())->GetBinContent(b);
   // calculate the phi weight wPhi for each bin:
   if(nParticlesInBin) wPhi = nParticlesPerBin/nParticlesInBin;
   (commonHist->GetHistPhiInt())->SetBinContent(b,wPhi);
  }
  listWeights->Add(commonHist->GetHistPhiInt());
 }else{cout<<" WARNING: the common control histos from the 1st run were not accessed."<<endl;}

 // common results histos:
 if(commonHistRes)
 {
  // diff. flow (pt):
  (commonHistRes->GetHistDiffFlowPtPOI())->SetName("pt_weights");
  if(!useFunctionOfPt) listWeights->Add(commonHistRes->GetHistDiffFlowPtPOI());
  // diff. flow (eta):
  (commonHistRes->GetHistDiffFlowEtaPOI())->SetName("eta_weights");
  listWeights->Add(commonHistRes->GetHistDiffFlowEtaPOI());
 }else{cout<<" WARNING: the common results histos from the 1st run were not accessed."<<endl;}

 // pt weights linear and quadratic in pt:
 if(useFunctionOfPt)
 {
  Double_t ptMin = AliFlowCommonConstants::GetPtMin();
  Double_t ptMax = AliFlowCommonConstants::GetPtMax();
  Int_t nBinsPt  = AliFlowCommonConstants::GetNbinsPt();
  Double_t ptCutOff = 4.0; // for pt > ptCutOff use constant weights
  if(nBinsPt==0)
  {
   cout<<" WARNING: number of pt bins is 0. "<<endl;
   exit(0);
  }
  Double_t ptBinWidth = 1.*(ptMax-ptMin)/nBinsPt;

  TH1D *ptFunction = new TH1D("ptFunction", "ptFunction", nBinsPt, ptMin, ptMax);
  ptFunction->SetName("pt_weights");
  ptFunction->SetXTitle("p_{t} [GeV]");
  ptFunction->SetYTitle("weights");
  Int_t power = 0;
  if(useLinearPt)
  {
   power = 1;
   ptFunction->SetTitle("p_{t} weights: #propto p_{t}");
  }
  if(useQuadraticPt)
  {
   power = 2;
   ptFunction->SetTitle("p_{t} weights: #propto p_{t}^{2}");
  }
  for(Int_t b=1;b<nBinsPt+1;b++)
  {
   if(ptMin+b*ptBinWidth < ptCutOff)
   {
    ptFunction->SetBinContent(b, pow(ptMin+b*ptBinWidth, power));
   }else
    {
     ptFunction->SetBinContent(b, pow(ptCutOff, power));
    }
  }
  listWeights->Add(ptFunction);
 }

 outputFile->WriteObject(listWeights,"weights","SingleKey");

 delete listWeights;
 delete outputFile;
}

<|endoftext|>


<commit_before><commit_msg>Display the message for game over.<commit_after><|endoftext|>


<commit_before>/** sql_functions.cc
    Jeremy Barnes, 6 January 2015
    Copyright (c) 2015 Datacratic Inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.
*/

#include "sql_functions.h"
#include "mldb/server/mldb_server.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/server/dataset_context.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/jml/utils/worker_task.h"
#include "mldb/server/function_contexts.h"
#include "mldb/server/bound_queries.h"
#include "mldb/sql/table_expression_operations.h"
#include "mldb/sql/join_utils.h"
#include "mldb/sql/execution_pipeline.h"
#include "mldb/arch/backtrace.h"
#include "mldb/types/any_impl.h"
#include "mldb/server/per_thread_accumulator.h"
#include "mldb/rest/in_process_rest_connection.h"
#include <memory>

using namespace std;

namespace Datacratic {
namespace MLDB {

std::shared_ptr<PipelineElement>
getMldbRoot(MldbServer * server)
{
    return PipelineElement::root(std::make_shared<SqlExpressionMldbContext>(server));
}

/*****************************************************************************/
/* SQL QUERY FUNCTION                                                        */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(SqlQueryFunctionConfig);

SqlQueryFunctionConfigDescription::
SqlQueryFunctionConfigDescription()
{
    addField("query", &SqlQueryFunctionConfig::query,
             "SQL query to run.  The values in the dataset, as "
             "well as the input values, will be available for the expression "
             "calculation");
}

SqlQueryFunction::
SqlQueryFunction(MldbServer * owner,
                 PolyConfig config,
                 const std::function<bool (const Json::Value &)> & onProgress)
    : Function(owner)
{
    functionConfig = config.params.convert<SqlQueryFunctionConfig>();
}

Any
SqlQueryFunction::
getStatus() const
{
    Json::Value result;
    result["expression"]["query"]["surface"] = functionConfig.query.stm->surface;
    result["expression"]["query"]["ast"] = functionConfig.query.stm->print();
    return result;
}

/** Structure that does all the work of the SQL expression function. */
struct SqlQueryFunctionApplier: public FunctionApplier {
    SqlQueryFunctionApplier(const SqlQueryFunction * function,
                            const SqlQueryFunctionConfig & config)
        : FunctionApplier(function),
          from(std::move(from))
    {
        // Called when we bind a parameter, to get its information
        auto getParamInfo = [&] (const Utf8String & paramName)
            {
                auto info = std::make_shared<AnyValueInfo>();

                // Record that we need it into our input info
                this->info.input.addValue(paramName, info);
                return info;
            };

        if (!config.query.stm->groupBy.empty()) {
            // Create our pipeline

            pipeline
                = getMldbRoot(function->server)
                ->params(getParamInfo)
                ->from(config.query.stm->from, config.query.stm->when,
                       SelectExpression::STAR, config.query.stm->where)
                ->where(config.query.stm->where)
                ->select(config.query.stm->groupBy)
                ->sort(config.query.stm->groupBy)
                ->partition(config.query.stm->groupBy.clauses.size())
                ->where(config.query.stm->having)
                ->select(config.query.stm->orderBy)
                ->sort(config.query.stm->orderBy)
                ->select(config.query.stm->select);
        }
        else {

            // Create our pipeline
            pipeline
                = getMldbRoot(function->server)
                ->params(getParamInfo)
                ->from(config.query.stm->from, config.query.stm->when,
                       SelectExpression::STAR, config.query.stm->where)
                ->where(config.query.stm->where)
                ->select(config.query.stm->orderBy)
                ->sort(config.query.stm->orderBy)
                ->select(config.query.stm->select);
        }

        // Bind the pipeline
        boundPipeline = pipeline->bind();

        // What type does the pipeline return?
        this->info.output = *boundPipeline->outputScope()->outputInfo().back();
    }

    virtual ~SqlQueryFunctionApplier()
    {
    }

    FunctionOutput apply(const FunctionContext & context) const
    {
        // 1.  Run our generator, finding all rows
        BoundParameters params
            = [&] (const Utf8String & name) -> ExpressionValue
            {
                return context.get(name);
            };

        auto executor = boundPipeline->start(params,
                                             !QueryThreadTracker::inChildThread() /* allowParallel */);
        auto output = executor->take();

        //if (output)
        //    cerr << "got output " << jsonEncode(output) << endl;

        FunctionOutput result;
        if (output)
            result = std::move(output->values.back());
        return result;
    }

    std::shared_ptr<Dataset> from;
    std::shared_ptr<PipelineElement> pipeline;
    std::shared_ptr<BoundPipelineElement> boundPipeline;
};

std::unique_ptr<FunctionApplier>
SqlQueryFunction::
bind(SqlBindingScope & outerContext,
     const FunctionValues & input) const
{
    std::unique_ptr<SqlQueryFunctionApplier> result
        (new SqlQueryFunctionApplier(this, functionConfig));

    // Check that these input values can provide everything needed for the result
    input.checkCompatibleAsInputTo(result->info.input);

    return std::move(result);
}

FunctionOutput
SqlQueryFunction::
apply(const FunctionApplier & applier,
      const FunctionContext & context) const
{
    return static_cast<const SqlQueryFunctionApplier &>(applier)
        .apply(context);
}

FunctionInfo
SqlQueryFunction::
getFunctionInfo() const
{
    SqlQueryFunctionApplier applier(this, functionConfig);
    return applier.info;
}

static RegisterFunctionType<SqlQueryFunction, SqlQueryFunctionConfig>
regSqlQueryFunction(builtinPackage(),
                    "sql.query",
                    "Run a single row SQL query against a dataset",
                    "functions/SqlQueryFunction.md.html");

/*****************************************************************************/
/* SQL EXPRESSION FUNCTION                                                   */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(SqlExpressionFunctionConfig);

SqlExpressionFunctionConfigDescription::
SqlExpressionFunctionConfigDescription()
{
    addField("expression", &SqlExpressionFunctionConfig::expression,
             "SQL expression function to run.  Takes the same syntax as a SELECT "
             "clause (but without the SELECT keyword); for example "
             "'x, y + 1 AS z'");
    addField("prepared", &SqlExpressionFunctionConfig::prepared,
             "Do we pre-prepare the expression to be run many times quickly?  "
             "If this is true, it will only be bound once, for generic "
             "inputs, and so will allow for quick individual queries, "
             "possibly at the expense of batch queries being slower.  "
             "If this is false, the default, then for every query the "
             "expression will be specialized for that query's data type. "
             "This can lead to faster batch queries, at the expense of a "
             "possibly high per-query overhead for individual queries.");
}

SqlExpressionFunction::
SqlExpressionFunction(MldbServer * owner,
                      PolyConfig config,
                      const std::function<bool (const Json::Value &)> & onProgress)
    : Function(owner), outerScope(owner), innerScope(outerScope)
{
    functionConfig = config.params.convert<SqlExpressionFunctionConfig>();

    // 1.  Bind the expression in.  That will tell us what it is expecting
    //     as an input.
    this->bound = functionConfig.expression.bind(innerScope);

    // 2.  Our output is known by the bound expression
    info.output = *bound.info;

    // 3.  Our required input is known by the binding context, as it records
    //     what was read.
    info.input = innerScope.input;

}

Any
SqlExpressionFunction::
getStatus() const
{
    Json::Value result;
    result["expression"]["surface"] = functionConfig.expression.surface;
    result["expression"]["ast"] = functionConfig.expression.print();
    return result;
}

/** Structure that does all the work of the SQL expression function. */
struct SqlExpressionFunctionApplier: public FunctionApplier {
    SqlExpressionFunctionApplier(SqlBindingScope & outerContext,
                                 const SqlExpressionFunction * function,
                                 const FunctionValues & input)
        : FunctionApplier(function), function(function),
          innerScope(const_cast<SqlExpressionMldbContext &>
                     (function->outerScope), input)
    {
        this->info = function->info;
        if (!function->functionConfig.prepared) {
            // Specialize to this input
            this->bound = function->functionConfig.expression.bind(innerScope);
            // That leads to a specialized output
            info.output = *bound.info;
        }
    }

    virtual ~SqlExpressionFunctionApplier()
    {
    }

    FunctionOutput apply(const SqlRowScope& outerRowScope,
                         const FunctionContext & context) const
    {
        if (function->functionConfig.prepared) {
            // Use the pre-bound version
            return function->bound(function->innerScope.getRowContext(outerRowScope, context));
        }
        else {
            // Use the specialized version
            return bound(this->innerScope.getRowContext(outerRowScope, context));
        }
    }

    const SqlExpressionFunction * function;
    FunctionExpressionContext innerScope;
    BoundSqlExpression bound;
};

std::unique_ptr<FunctionApplier>
SqlExpressionFunction::
bind(SqlBindingScope & outerContext,
     const FunctionValues & input) const
{
    std::unique_ptr<SqlExpressionFunctionApplier> result
        (new SqlExpressionFunctionApplier(outerContext, this, input));

    // Check that these input values can provide everything needed for the result
    input.checkCompatibleAsInputTo(result->info.input);

    return std::move(result);
}

FunctionOutput
SqlExpressionFunction::
applyOuter(const SqlRowScope& outer, const FunctionApplier & applier,
                              const FunctionContext & context) const
{
  return static_cast<const SqlExpressionFunctionApplier &>(applier)
        .apply(outer, context);
}

FunctionOutput
SqlExpressionFunction::
apply(const FunctionApplier & applier,
      const FunctionContext & context) const
{
    //applyOuter should get called instead
    ExcAssert(false);
}

FunctionInfo
SqlExpressionFunction::
getFunctionInfo() const
{
    return this->info;
    FunctionInfo result;

    return result;
}

static RegisterFunctionType<SqlExpressionFunction, SqlExpressionFunctionConfig>
regSqlExpressionFunction(builtinPackage(),
                         "sql.expression",
                         "Run an SQL expression as a function",
                         "functions/SqlExpressionFunction.md.html");

/*****************************************************************************/
/* TRANSFORM DATASET                                                         */
/*****************************************************************************/

TransformDatasetConfig::
TransformDatasetConfig()
    : skipEmptyRows(false)
{
    outputDataset.withType("sparse.mutable");
}

DEFINE_STRUCTURE_DESCRIPTION(TransformDatasetConfig);

TransformDatasetConfigDescription::
TransformDatasetConfigDescription()
{
    addField("inputData", &TransformDatasetConfig::inputData,
             "A SQL statement to select the rows from a dataset to be transformed.  This supports "
             "all MLDB's SQL expressions including but not limited to where, when, order by and "
             "group by clauses.  These expressions can be used to refine the rows to transform.");
    addField("outputDataset", &TransformDatasetConfig::outputDataset,
             "Output dataset configuration.  This may refer either to an "
             "existing dataset, or a fully specified but non-existing dataset "
             "which will be created by the procedure.", PolyConfigT<Dataset>().withType("sparse.mutable"));
    addField("skipEmptyRows", &TransformDatasetConfig::skipEmptyRows,
             "Skip rows from the input dataset where no values are selected",
             false);
    addParent<ProcedureConfig>();
}

TransformDataset::
TransformDataset(MldbServer * owner,
                 PolyConfig config,
                 const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    procedureConfig = config.params.convert<TransformDatasetConfig>();
}

RunOutput
TransformDataset::
run(const ProcedureRunConfig & run,
    const std::function<bool (const Json::Value &)> & onProgress) const
{
    // Get the input dataset
    SqlExpressionMldbContext context(server);

    auto boundDataset = procedureConfig.inputData.stm->from->bind(context);
    std::vector< std::shared_ptr<SqlExpression> > aggregators =
        procedureConfig.inputData.stm->select.findAggregators();

    // Create the output
    std::shared_ptr<Dataset> output;
    if (!procedureConfig.outputDataset.type.empty() || !procedureConfig.outputDataset.id.empty()) {
        output = createDataset(server, procedureConfig.outputDataset, nullptr, true /*overwrite*/);
    }

    bool skipEmptyRows = procedureConfig.skipEmptyRows;

    // Run it
    if (procedureConfig.inputData.stm->groupBy.clauses.empty()) {

        // We accumulate multiple rows per thread and insert with recordRows
        // to be more efficient.
        PerThreadAccumulator<std::vector<std::pair<RowName, std::vector<std::tuple<ColumnName, CellValue, Date> > > > > accum;

        auto recordRowInOutputDataset
            = [&] (const MatrixNamedRow & row,
                   const std::vector<ExpressionValue> & calc)
            {
                // Nulls with non-finite timestamp are not recorded; they
                // come from an expression that matched nothing and can't
                // be represented (they will be read automatically as nulls).
                std::vector<std::tuple<ColumnName, CellValue, Date> > cols;
                cols.reserve(row.columns.size());
                for (auto & c: row.columns) {
                    if (std::get<1>(c).empty()
                        && !std::get<2>(c).isADate())
                        continue;
                    cols.push_back(c);
                }

                if (!skipEmptyRows || cols.size() > 0)
                {
                    auto & rows = accum.get();
                    rows.reserve(10000);
                    rows.emplace_back(RowName(calc.at(0).toString()), std::move(cols));

                    if (rows.size() >= 10000) {
                        output->recordRows(rows);
                        rows.clear();
                    }
                }

                return true;
            };

        // We only add an implicit order by (which defeats parallelization)
        // if we have a limit or offset parameter.
        bool implicitOrderByRowHash
            = (procedureConfig.inputData.stm->offset != 0 ||
               procedureConfig.inputData.stm->limit != -1);

        BoundSelectQuery(procedureConfig.inputData.stm->select,
                         *boundDataset.dataset,
                         boundDataset.asName,
                         procedureConfig.inputData.stm->when,
                         procedureConfig.inputData.stm->where,
                         procedureConfig.inputData.stm->orderBy,
                         { procedureConfig.inputData.stm->rowName },
                         implicitOrderByRowHash)
            .execute(recordRowInOutputDataset,
                     procedureConfig.inputData.stm->offset,
                     procedureConfig.inputData.stm->limit,
                     onProgress);

        // Finish off the last bits of each thread
        accum.forEach([&] (std::vector<std::pair<RowName, std::vector<std::tuple<ColumnName, CellValue, Date> > > > * rows)
                      {
                          output->recordRows(*rows);
                      });
    }
    else {
        auto recordRowInOutputDataset
            = [&] (const MatrixNamedRow & row)
            {
                output->recordRow(row.rowName, row.columns);
                return true;
            };

        BoundGroupByQuery(procedureConfig.inputData.stm->select,
                          *boundDataset.dataset,
                          boundDataset.asName,
                          procedureConfig.inputData.stm->when,
                          procedureConfig.inputData.stm->where,
                          procedureConfig.inputData.stm->groupBy,
                          aggregators,
                          *procedureConfig.inputData.stm->having,
                          *procedureConfig.inputData.stm->rowName,
                          procedureConfig.inputData.stm->orderBy)
            .execute(recordRowInOutputDataset,
                     procedureConfig.inputData.stm->offset,
                     procedureConfig.inputData.stm->limit,
                     onProgress);
    }
    // Save the dataset we created

    output->commit();

    return output->getStatus();
}

Any
TransformDataset::
getStatus() const
{
    return Any();
}

static RegisterProcedureType<TransformDataset, TransformDatasetConfig>
regTransformDataset(builtinPackage(),
                    "transform",
                    "Apply an SQL expression over a dataset to transform into another dataset",
                    "procedures/TransformDataset.md.html");

} // namespace MLDB
} // namespace Datacratic
<commit_msg>Fixed detection of infinite function recursion (MLDB-1293)<commit_after>/** sql_functions.cc
    Jeremy Barnes, 6 January 2015
    Copyright (c) 2015 Datacratic Inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved.
*/

#include "sql_functions.h"
#include "mldb/server/mldb_server.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/server/dataset_context.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/jml/utils/worker_task.h"
#include "mldb/server/function_contexts.h"
#include "mldb/server/bound_queries.h"
#include "mldb/sql/table_expression_operations.h"
#include "mldb/sql/join_utils.h"
#include "mldb/sql/execution_pipeline.h"
#include "mldb/arch/backtrace.h"
#include "mldb/types/any_impl.h"
#include "mldb/server/per_thread_accumulator.h"
#include "mldb/rest/in_process_rest_connection.h"
#include <memory>

using namespace std;

namespace Datacratic {
namespace MLDB {

std::shared_ptr<PipelineElement>
getMldbRoot(MldbServer * server)
{
    return PipelineElement::root(std::make_shared<SqlExpressionMldbContext>(server));
}

/*****************************************************************************/
/* SQL QUERY FUNCTION                                                        */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(SqlQueryFunctionConfig);

SqlQueryFunctionConfigDescription::
SqlQueryFunctionConfigDescription()
{
    addField("query", &SqlQueryFunctionConfig::query,
             "SQL query to run.  The values in the dataset, as "
             "well as the input values, will be available for the expression "
             "calculation");
}

SqlQueryFunction::
SqlQueryFunction(MldbServer * owner,
                 PolyConfig config,
                 const std::function<bool (const Json::Value &)> & onProgress)
    : Function(owner)
{
    functionConfig = config.params.convert<SqlQueryFunctionConfig>();
}

Any
SqlQueryFunction::
getStatus() const
{
    Json::Value result;
    result["expression"]["query"]["surface"] = functionConfig.query.stm->surface;
    result["expression"]["query"]["ast"] = functionConfig.query.stm->print();
    return result;
}

/** Structure that does all the work of the SQL expression function. */
struct SqlQueryFunctionApplier: public FunctionApplier {
    SqlQueryFunctionApplier(const SqlQueryFunction * function,
                            const SqlQueryFunctionConfig & config)
        : FunctionApplier(function),
          from(std::move(from))
    {
        // Called when we bind a parameter, to get its information
        auto getParamInfo = [&] (const Utf8String & paramName)
            {
                auto info = std::make_shared<AnyValueInfo>();

                // Record that we need it into our input info
                this->info.input.addValue(paramName, info);
                return info;
            };

        if (!config.query.stm->groupBy.empty()) {
            // Create our pipeline

            pipeline
                = getMldbRoot(function->server)
                ->params(getParamInfo)
                ->from(config.query.stm->from, config.query.stm->when,
                       SelectExpression::STAR, config.query.stm->where)
                ->where(config.query.stm->where)
                ->select(config.query.stm->groupBy)
                ->sort(config.query.stm->groupBy)
                ->partition(config.query.stm->groupBy.clauses.size())
                ->where(config.query.stm->having)
                ->select(config.query.stm->orderBy)
                ->sort(config.query.stm->orderBy)
                ->select(config.query.stm->select);
        }
        else {

            // Create our pipeline
            pipeline
                = getMldbRoot(function->server)
                ->params(getParamInfo)
                ->from(config.query.stm->from, config.query.stm->when,
                       SelectExpression::STAR, config.query.stm->where)
                ->where(config.query.stm->where)
                ->select(config.query.stm->orderBy)
                ->sort(config.query.stm->orderBy)
                ->select(config.query.stm->select);
        }

        // Bind the pipeline
        boundPipeline = pipeline->bind();

        // What type does the pipeline return?
        this->info.output = *boundPipeline->outputScope()->outputInfo().back();
    }

    virtual ~SqlQueryFunctionApplier()
    {
    }

    FunctionOutput apply(const FunctionContext & context) const
    {
        // 1.  Run our generator, finding all rows
        BoundParameters params
            = [&] (const Utf8String & name) -> ExpressionValue
            {
                return context.get(name);
            };

        auto executor = boundPipeline->start(params,
                                             !QueryThreadTracker::inChildThread() /* allowParallel */);
        auto output = executor->take();

        //if (output)
        //    cerr << "got output " << jsonEncode(output) << endl;

        FunctionOutput result;
        if (output)
            result = std::move(output->values.back());
        return result;
    }

    std::shared_ptr<Dataset> from;
    std::shared_ptr<PipelineElement> pipeline;
    std::shared_ptr<BoundPipelineElement> boundPipeline;
};

std::unique_ptr<FunctionApplier>
SqlQueryFunction::
bind(SqlBindingScope & outerContext,
     const FunctionValues & input) const
{
    std::unique_ptr<SqlQueryFunctionApplier> result
        (new SqlQueryFunctionApplier(this, functionConfig));

    // Check that these input values can provide everything needed for the result
    input.checkCompatibleAsInputTo(result->info.input);

    return std::move(result);
}

FunctionOutput
SqlQueryFunction::
apply(const FunctionApplier & applier,
      const FunctionContext & context) const
{
    return static_cast<const SqlQueryFunctionApplier &>(applier)
        .apply(context);
}

FunctionInfo
SqlQueryFunction::
getFunctionInfo() const
{
    SqlQueryFunctionApplier applier(this, functionConfig);
    return applier.info;
}

static RegisterFunctionType<SqlQueryFunction, SqlQueryFunctionConfig>
regSqlQueryFunction(builtinPackage(),
                    "sql.query",
                    "Run a single row SQL query against a dataset",
                    "functions/SqlQueryFunction.md.html");

/*****************************************************************************/
/* SQL EXPRESSION FUNCTION                                                   */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(SqlExpressionFunctionConfig);

SqlExpressionFunctionConfigDescription::
SqlExpressionFunctionConfigDescription()
{
    addField("expression", &SqlExpressionFunctionConfig::expression,
             "SQL expression function to run.  Takes the same syntax as a SELECT "
             "clause (but without the SELECT keyword); for example "
             "'x, y + 1 AS z'");
    addField("prepared", &SqlExpressionFunctionConfig::prepared,
             "Do we pre-prepare the expression to be run many times quickly?  "
             "If this is true, it will only be bound once, for generic "
             "inputs, and so will allow for quick individual queries, "
             "possibly at the expense of batch queries being slower.  "
             "If this is false, the default, then for every query the "
             "expression will be specialized for that query's data type. "
             "This can lead to faster batch queries, at the expense of a "
             "possibly high per-query overhead for individual queries.");
}

SqlExpressionFunction::
SqlExpressionFunction(MldbServer * owner,
                      PolyConfig config,
                      const std::function<bool (const Json::Value &)> & onProgress)
    : Function(owner), outerScope(owner), innerScope(outerScope)
{
    functionConfig = config.params.convert<SqlExpressionFunctionConfig>();

    if (functionConfig.prepared) {
        // 1.  Bind the expression in.  That will tell us what it is expecting
        //     as an input.
        this->bound = functionConfig.expression.bind(innerScope);

        // 2.  Our output is known by the bound expression
        this->info.output = *this->bound.info;

        // 3.  Our required input is known by the binding context, as it records
        //     what was read.
        info.input = innerScope.input;
    }
}

Any
SqlExpressionFunction::
getStatus() const
{
    Json::Value result;
    result["expression"]["surface"] = functionConfig.expression.surface;
    result["expression"]["ast"] = functionConfig.expression.print();
    return result;
}

/** Structure that does all the work of the SQL expression function. */
struct SqlExpressionFunctionApplier: public FunctionApplier {
    SqlExpressionFunctionApplier(SqlBindingScope & outerScope,
                                 const SqlExpressionFunction * function,
                                 const FunctionValues & input)
        : FunctionApplier(function),
          function(function),
          innerScope(outerScope, input)
    {
        if (!function->functionConfig.prepared) {
            // Specialize to this input
            this->bound = function->functionConfig.expression.bind(innerScope);
            // That leads to a specialized output
            this->info.output = *bound.info;
        }
        else {
            this->info = function->info;
        }
    }

    virtual ~SqlExpressionFunctionApplier()
    {
    }

    FunctionOutput apply(const SqlRowScope& outerRowScope,
                         const FunctionContext & context) const
    {
        if (function->functionConfig.prepared) {
            // Use the pre-bound version.    Note that we ignore the outer
            // row scope, which wasn't available when we prepared the
            // expression.
            SqlRowScope emptyOuterRowScope;
            return function->bound(function->innerScope.getRowContext(emptyOuterRowScope, context));
        }
        else {
            // Use the specialized version.  The outer row scope is available
            // from the expression.
            return bound(this->innerScope.getRowContext(outerRowScope, context));
        }
    }

    const SqlExpressionFunction * function;
    FunctionExpressionContext innerScope;
    BoundSqlExpression bound;
};

std::unique_ptr<FunctionApplier>
SqlExpressionFunction::
bind(SqlBindingScope & outerContext,
     const FunctionValues & input) const
{
    std::unique_ptr<SqlExpressionFunctionApplier> result
        (new SqlExpressionFunctionApplier(outerContext, this, input));

    // Check that these input values can provide everything needed for the result
    input.checkCompatibleAsInputTo(result->info.input);

    return std::move(result);
}

FunctionOutput
SqlExpressionFunction::
applyOuter(const SqlRowScope& outer, const FunctionApplier & applier,
                              const FunctionContext & context) const
{
  return static_cast<const SqlExpressionFunctionApplier &>(applier)
        .apply(outer, context);
}

FunctionOutput
SqlExpressionFunction::
apply(const FunctionApplier & applier,
      const FunctionContext & context) const
{
    //applyOuter should get called instead
    ExcAssert(false);
}

FunctionInfo
SqlExpressionFunction::
getFunctionInfo() const
{
    if (functionConfig.prepared) {
        return this->info;
    }

    FunctionInfo result;

    // 0.  We want the pure function information, so we assume there is
    //     no context for it apart from MLDB itself.
    SqlExpressionMldbContext outerContext(MldbEntity::getOwner(this->server));

    // 1.  Create a binding context to see what this function takes
    FunctionExpressionContext context(outerContext);

    // 2.  Bind the expression in.  That will tell us what it is expecting
    //     as an input.
    BoundSqlExpression bound = functionConfig.expression.bind(context);

    // 3.  Our output is known by the bound expression
    result.output = *bound.info;

    // 4.  Our required input is known by the binding context, as it records
    //     what was read.
    result.input = context.input;

    return result;
}

static RegisterFunctionType<SqlExpressionFunction, SqlExpressionFunctionConfig>
regSqlExpressionFunction(builtinPackage(),
                         "sql.expression",
                         "Run an SQL expression as a function",
                         "functions/SqlExpressionFunction.md.html");

/*****************************************************************************/
/* TRANSFORM DATASET                                                         */
/*****************************************************************************/

TransformDatasetConfig::
TransformDatasetConfig()
    : skipEmptyRows(false)
{
    outputDataset.withType("sparse.mutable");
}

DEFINE_STRUCTURE_DESCRIPTION(TransformDatasetConfig);

TransformDatasetConfigDescription::
TransformDatasetConfigDescription()
{
    addField("inputData", &TransformDatasetConfig::inputData,
             "A SQL statement to select the rows from a dataset to be transformed.  This supports "
             "all MLDB's SQL expressions including but not limited to where, when, order by and "
             "group by clauses.  These expressions can be used to refine the rows to transform.");
    addField("outputDataset", &TransformDatasetConfig::outputDataset,
             "Output dataset configuration.  This may refer either to an "
             "existing dataset, or a fully specified but non-existing dataset "
             "which will be created by the procedure.", PolyConfigT<Dataset>().withType("sparse.mutable"));
    addField("skipEmptyRows", &TransformDatasetConfig::skipEmptyRows,
             "Skip rows from the input dataset where no values are selected",
             false);
    addParent<ProcedureConfig>();
}

TransformDataset::
TransformDataset(MldbServer * owner,
                 PolyConfig config,
                 const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    procedureConfig = config.params.convert<TransformDatasetConfig>();
}

RunOutput
TransformDataset::
run(const ProcedureRunConfig & run,
    const std::function<bool (const Json::Value &)> & onProgress) const
{
    // Get the input dataset
    SqlExpressionMldbContext context(server);

    auto boundDataset = procedureConfig.inputData.stm->from->bind(context);
    std::vector< std::shared_ptr<SqlExpression> > aggregators =
        procedureConfig.inputData.stm->select.findAggregators();

    // Create the output
    std::shared_ptr<Dataset> output;
    if (!procedureConfig.outputDataset.type.empty() || !procedureConfig.outputDataset.id.empty()) {
        output = createDataset(server, procedureConfig.outputDataset, nullptr, true /*overwrite*/);
    }

    bool skipEmptyRows = procedureConfig.skipEmptyRows;

    // Run it
    if (procedureConfig.inputData.stm->groupBy.clauses.empty()) {

        // We accumulate multiple rows per thread and insert with recordRows
        // to be more efficient.
        PerThreadAccumulator<std::vector<std::pair<RowName, std::vector<std::tuple<ColumnName, CellValue, Date> > > > > accum;

        auto recordRowInOutputDataset
            = [&] (const MatrixNamedRow & row,
                   const std::vector<ExpressionValue> & calc)
            {
                // Nulls with non-finite timestamp are not recorded; they
                // come from an expression that matched nothing and can't
                // be represented (they will be read automatically as nulls).
                std::vector<std::tuple<ColumnName, CellValue, Date> > cols;
                cols.reserve(row.columns.size());
                for (auto & c: row.columns) {
                    if (std::get<1>(c).empty()
                        && !std::get<2>(c).isADate())
                        continue;
                    cols.push_back(c);
                }

                if (!skipEmptyRows || cols.size() > 0)
                {
                    auto & rows = accum.get();
                    rows.reserve(10000);
                    rows.emplace_back(RowName(calc.at(0).toString()), std::move(cols));

                    if (rows.size() >= 10000) {
                        output->recordRows(rows);
                        rows.clear();
                    }
                }

                return true;
            };

        // We only add an implicit order by (which defeats parallelization)
        // if we have a limit or offset parameter.
        bool implicitOrderByRowHash
            = (procedureConfig.inputData.stm->offset != 0 ||
               procedureConfig.inputData.stm->limit != -1);

        BoundSelectQuery(procedureConfig.inputData.stm->select,
                         *boundDataset.dataset,
                         boundDataset.asName,
                         procedureConfig.inputData.stm->when,
                         procedureConfig.inputData.stm->where,
                         procedureConfig.inputData.stm->orderBy,
                         { procedureConfig.inputData.stm->rowName },
                         implicitOrderByRowHash)
            .execute(recordRowInOutputDataset,
                     procedureConfig.inputData.stm->offset,
                     procedureConfig.inputData.stm->limit,
                     onProgress);

        // Finish off the last bits of each thread
        accum.forEach([&] (std::vector<std::pair<RowName, std::vector<std::tuple<ColumnName, CellValue, Date> > > > * rows)
                      {
                          output->recordRows(*rows);
                      });
    }
    else {
        auto recordRowInOutputDataset
            = [&] (const MatrixNamedRow & row)
            {
                output->recordRow(row.rowName, row.columns);
                return true;
            };

        BoundGroupByQuery(procedureConfig.inputData.stm->select,
                          *boundDataset.dataset,
                          boundDataset.asName,
                          procedureConfig.inputData.stm->when,
                          procedureConfig.inputData.stm->where,
                          procedureConfig.inputData.stm->groupBy,
                          aggregators,
                          *procedureConfig.inputData.stm->having,
                          *procedureConfig.inputData.stm->rowName,
                          procedureConfig.inputData.stm->orderBy)
            .execute(recordRowInOutputDataset,
                     procedureConfig.inputData.stm->offset,
                     procedureConfig.inputData.stm->limit,
                     onProgress);
    }
    // Save the dataset we created

    output->commit();

    return output->getStatus();
}

Any
TransformDataset::
getStatus() const
{
    return Any();
}

static RegisterProcedureType<TransformDataset, TransformDatasetConfig>
regTransformDataset(builtinPackage(),
                    "transform",
                    "Apply an SQL expression over a dataset to transform into another dataset",
                    "procedures/TransformDataset.md.html");

} // namespace MLDB
} // namespace Datacratic
<|endoftext|>


<commit_before>#include <cmath>
#include <cstdio>
#include <vector>
#include <iostream>
#include <algorithm>
#include <bitset>
#include <string.h>
using namespace std;

int main() {
    int t; cin >> t;
    vector<string> answers(t);

    for (int i = 0; i < t; i++) {
        int x; cin >> x;
        string binary = bitset<32>(x).to_string();
        answers[i] = binary;
    }
    for (string& x : answers) {
    	// the problem wants us to remove leading zeros:
    	// thus, we must find the first 1 in the binary number
        int start_printing = 0;
        string zero = "0"; // make a string to use in comparisons
        for (int i = 0, n = x.size(); i < n; i++) {
            if (zero[0] == x[i]) continue; // i want to be explicit about continuing
            else {
                start_printing = i; // saves index of first 1
                break; // breaks the loop to escape overwriting start_printing
            }
        }

        for (int i = start_printing, n = x.size(); i < n; i++)
            cout << x[i];
        cout << endl;
    }
    return 0;
}<commit_msg>add day 10<commit_after>#include <cmath>
#include <cstdio>
#include <vector>
#include <iostream>
#include <bitset>
#include <string.h>
using namespace std;

int main() {
    int t; cin >> t;
    vector<string> answers(t);

    for (int i = 0; i < t; i++) {
        int x; cin >> x;
        string binary = bitset<32>(x).to_string();
        answers[i] = binary;
    }
    for (string& x : answers) {
    	// the problem wants us to remove leading zeros:
    	// thus, we must find the first 1 in the binary number
        int start_printing = 0;
        string zero = "0"; // make a string to use in comparisons
        for (int i = 0, n = x.size(); i < n; i++) {
            if (zero[0] == x[i]) continue; // i want to be explicit about continuing
            else {
                start_printing = i; // saves index of first 1
                break; // breaks the loop to escape overwriting start_printing
            }
        }

        for (int i = start_printing, n = x.size(); i < n; i++)
            cout << x[i];
        cout << endl;
    }
    return 0;
}<|endoftext|>


<commit_before>//
// Copyright (c) articy Software GmbH & Co. KG. All rights reserved.
//

#include "Slate/Pins/SArticyRefPin.h"
#include "ArticyRef.h"
#include "EdGraph/EdGraphSchema.h"
#include "EdGraph/EdGraphNode.h"
#include "Slate/SArticyIdProperty.h"
#include "ScopedTransaction.h"
#include "Widgets/Input/SMultiLineEditableTextBox.h"
#include "Slate/SArticyRefProperty.h"

void SArticyRefPin::Construct(const FArguments& InArgs, UEdGraphPin* GraphPin)
{
	SGraphPin::Construct(SGraphPin::FArguments(), GraphPin);
}

TSharedRef<SWidget> SArticyRefPin::GetDefaultValueWidget()
{
	return SNew(SArticyRefProperty)
		.ArticyRefToDisplay(this, &SArticyRefPin::GetArticyRef)
		.OnArticyRefChanged(this, &SArticyRefPin::OnArticyRefChanged)
		.bIsReadOnly(true)
		.Visibility(this, &SArticyRefPin::GetDefaultValueVisibility);
}

EVisibility SArticyRefPin::GetDefaultValueVisibility() const
{
	if (bOnlyShowDefaultValue)
	{
		return EVisibility::Visible;
	}

	const UEdGraphSchema* Schema = !GraphPinObj->IsPendingKill() ? GraphPinObj->GetSchema() : nullptr;
	if (Schema == nullptr || Schema->ShouldHidePinDefaultValue(GraphPinObj))
	{
		return EVisibility::Collapsed;
	}

	// If it's not connectable we want to always show it
	if (!GraphPinObj->bOrphanedPin && GraphPinObj->bNotConnectable)
	{
		return EVisibility::Visible;
	}

	// don't show on outputs
	if (GraphPinObj->Direction == EGPD_Output)
	{
		return EVisibility::Collapsed;
	}

	// show only when not connected
	return IsConnected() ? EVisibility::Collapsed : EVisibility::Visible;
}

FArticyRef SArticyRefPin::GetArticyRef() const
{
	FString RefString = GraphPinObj->GetDefaultAsString();
	FArticyRef Ref;

	if(!Ref.InitFromString(RefString))
	{
		Ref = FArticyRef();
	}

	return Ref;
}

void SArticyRefPin::OnArticyRefChanged(const FArticyRef& NewArticyRef)
{
	FString FormattedValueString;
	FArticyRef::StaticStruct()->ExportText(FormattedValueString, &ArticyRef, nullptr, nullptr, (PPF_Copy), nullptr);

	ArticyRef = NewArticyRef;

	if(GraphPinObj->GetDefaultAsString() != FormattedValueString)
	{
		const FScopedTransaction Transaction(NSLOCTEXT("GraphEditor", "ChangeArticyRefValue", "Change Articy Ref Value"));
		GraphPinObj->Modify();

		GraphPinObj->GetSchema()->TrySetDefaultValue(*GraphPinObj, ArticyRef.ToString());

		if (OwnerNodePtr.IsValid())
		{
			//OwnerNodePtr.Pin()->UpdateGraphNode();
		}
	}
}
<commit_msg>Fixed ArticyRefPin being read only<commit_after>//
// Copyright (c) articy Software GmbH & Co. KG. All rights reserved.
//

#include "Slate/Pins/SArticyRefPin.h"
#include "ArticyRef.h"
#include "EdGraph/EdGraphSchema.h"
#include "EdGraph/EdGraphNode.h"
#include "Slate/SArticyIdProperty.h"
#include "ScopedTransaction.h"
#include "Widgets/Input/SMultiLineEditableTextBox.h"
#include "Slate/SArticyRefProperty.h"

void SArticyRefPin::Construct(const FArguments& InArgs, UEdGraphPin* GraphPin)
{
	SGraphPin::Construct(SGraphPin::FArguments(), GraphPin);
}

TSharedRef<SWidget> SArticyRefPin::GetDefaultValueWidget()
{
	return SNew(SArticyRefProperty)
		.ArticyRefToDisplay(this, &SArticyRefPin::GetArticyRef)
		.OnArticyRefChanged(this, &SArticyRefPin::OnArticyRefChanged)
		.Visibility(this, &SArticyRefPin::GetDefaultValueVisibility);
}

EVisibility SArticyRefPin::GetDefaultValueVisibility() const
{
	if (bOnlyShowDefaultValue)
	{
		return EVisibility::Visible;
	}

	const UEdGraphSchema* Schema = !GraphPinObj->IsPendingKill() ? GraphPinObj->GetSchema() : nullptr;
	if (Schema == nullptr || Schema->ShouldHidePinDefaultValue(GraphPinObj))
	{
		return EVisibility::Collapsed;
	}

	// If it's not connectable we want to always show it
	if (!GraphPinObj->bOrphanedPin && GraphPinObj->bNotConnectable)
	{
		return EVisibility::Visible;
	}

	// don't show on outputs
	if (GraphPinObj->Direction == EGPD_Output)
	{
		return EVisibility::Collapsed;
	}

	// show only when not connected
	return IsConnected() ? EVisibility::Collapsed : EVisibility::Visible;
}

FArticyRef SArticyRefPin::GetArticyRef() const
{
	FString RefString = GraphPinObj->GetDefaultAsString();
	FArticyRef Ref;

	if(!Ref.InitFromString(RefString))
	{
		Ref = FArticyRef();
	}

	return Ref;
}

void SArticyRefPin::OnArticyRefChanged(const FArticyRef& NewArticyRef)
{
	FString FormattedValueString;
	FArticyRef::StaticStruct()->ExportText(FormattedValueString, &ArticyRef, nullptr, nullptr, (PPF_Copy), nullptr);

	ArticyRef = NewArticyRef;

	if(GraphPinObj->GetDefaultAsString() != FormattedValueString)
	{
		const FScopedTransaction Transaction(NSLOCTEXT("GraphEditor", "ChangeArticyRefValue", "Change Articy Ref Value"));
		GraphPinObj->Modify();

		GraphPinObj->GetSchema()->TrySetDefaultValue(*GraphPinObj, ArticyRef.ToString());

		if (OwnerNodePtr.IsValid())
		{
			//OwnerNodePtr.Pin()->UpdateGraphNode();
		}
	}
}
<|endoftext|>