Return to httpclient.c CVS log

Up to [OMI] / omi / http

1.0.1

/*
**==============================================================================
**
** Open Management Infrastructure (OMI)
**
** Copyright (c) Microsoft Corporation
** 
** Licensed under the Apache License, Version 2.0 (the "License"); you may not 
** use this file except in compliance with the License. You may obtain a copy 
** of the License at 
**
**     http://www.apache.org/licenses/LICENSE-2.0 
**
** THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
** KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED 
** WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, 
** MERCHANTABLITY OR NON-INFRINGEMENT. 
**
** See the Apache 2 License for the specific language governing permissions 
** and limitations under the License.
**
**==============================================================================
*/

#include <assert.h>
#include <ctype.h>
//#include "http.h"
#include "httpclient.h"
#include <sock/addr.h>
#include <sock/sock.h>
#include <sock/selector.h>
#include <base/time.h>
#include <base/buf.h>
#include <base/log.h>
#include <base/result.h>
#include <base/strings.h>
#include <base/io.h>
#include <base/paths.h>

#ifdef CONFIG_POSIX
#include <openssl/ssl.h>
#include <openssl/err.h>
#else
/* ssl not supported in this configuration; just make compiler happy */
typedef void SSL;
typedef void SSL_CTX;
#define SSL_CTX_free(c)
#define SSL_new(c) 0
#define SSL_free(c)
#define SSL_set_connect_state(c)
#define SSL_set_fd(c,a) (a==a)
#define SSL_read(c,a,b) 0
#define SSL_write(c,a,b) 0
#define SSL_get_error(c,e) e
#define SSL_ERROR_WANT_WRITE 0
#define SSL_ERROR_WANT_READ 1
#define SSL_ERROR_SYSCALL 2

#ifdef EWOULDBLOCK
# undef EWOULDBLOCK
#endif

#define EWOULDBLOCK 0

#ifdef EINPROGRESS
# undef EINPROGRESS
#endif 

#define EINPROGRESS 0

#define ERR_get_error() 0
#define ERR_error_string_n(c,a,b) a[0]=0
#define SSL_accept(c) 0
#define SSL_connect(c) 0

#endif

#define T MI_T

/* #define ENABLE_TRACING  // */
#ifdef  ENABLE_TRACING
#define PRINTF(a)  TIMESTAMP(); printf a
#define TIMESTAMP() \
{\
        MI_Uint64 currentTimeUsec = 0;\
\
        Time_Now(&currentTimeUsec);\
        currentTimeUsec /= 1000;    /* ms */ \
        printf("%ds%03dms ", (int)(currentTimeUsec / 1000 % 1000), (int)(currentTimeUsec % 1000));\
}

#define PRINTF_2(a)
#else
#define PRINTF(a)
#define PRINTF_2(a)
#endif



/*
**==============================================================================
**
** Local definitions:
**
**==============================================================================
*/

static const MI_Uint32 _MAGIC = 0x5FC7B966;
static const MI_Uint32 MAX_HEADER_SIZE = 2 * 1024;
static const MI_Uint32 INITIAL_BUFFER_SIZE = 2 * 1024;
static const MI_Uint32 DEFAULT_HTTP_TIMEOUT_USEC = 60 * 1000000;

typedef enum _Http_RecvState
{
    RECV_STATE_HEADER,
    RECV_STATE_CONTENT,
    RECV_STATE_CHUNKHEADER,
    RECV_STATE_CHUNKDATA
}
Http_RecvState;

typedef struct _Http_SR_SocketData
{
    /* based member*/
    Handler     base;

    /* timeout */
    MI_Uint64   timeoutUsec;

    /* ssl part */
    SSL*  ssl;
    MI_Boolean  reverseOperations;  /*reverse read/write Events/Handlers*/
    MI_Boolean  connectDone;

    /* receiving data */
    char*       recvBuffer;
    size_t      recvBufferSize;
    size_t      recvievedSize;
    Http_RecvState      recvingState;
    HttpClientHeaderField recvHeaderFields[64];
    HttpClientResponseHeader   recvHeaders;
    MI_Sint64   contentLength;
    Page*   recvPage;

    /* sending part */
    Page*   sendPage;
    Page*   sendHeader;
    size_t      sentSize;
    Http_RecvState  sendingState;

    /* general operation status */
    MI_Result status;

}
Http_SR_SocketData;

struct _HttpClient
{
    MI_Uint32       magic;
    Selector        internalSelector;
    Selector*       selector;
    HttpClientCallbackOnStatus     callbackOnStatus;
    HttpClientCallbackOnResponse   callbackOnResponse;
    void*                               callbackData;
    SSL_CTX*    sslContext;

    Http_SR_SocketData* connector;

    MI_Boolean  internalSelectorUsed;
};


/* helper functions result */
typedef enum _Http_CallbackResult
{
    PRT_CONTINUE,
    PRT_RETURN_TRUE,
    PRT_RETURN_FALSE
}
Http_CallbackResult;


MI_INLINE MI_Uint8 _ToLower(MI_Uint8 x)
{
    return (MI_Uint8)tolower(x);
}

#define _HashCode(first,last,len) ( (((MI_Uint8)first) << 16) | (((MI_Uint8)last) << 8)  | (((MI_Uint16)len)) )

static MI_Boolean _getNameValuePair(
    char ** line,
    char ** value,
    int*  nameHashCode )
{
    int len = 0;
    char* p;
    /* find name end /hash-code */

    *nameHashCode =  _ToLower((MI_Uint8)(*line)[0])<<16;

    for (len = 1; (*line)[len] != ':' && (*line)[len] != '\r'; len++ )
        ;

    if ((*line)[len] != ':')
        return MI_FALSE;

    *nameHashCode |=  (len) | _ToLower((MI_Uint8)(*line)[len-1])<<8;
    (*line)[len] = 0;
    p = *line + len + 1;

    /* skip spaces in value */
    while (p[0] == ' ' || p[0] == '\t')
        p++;

    *value = p;

    /* skip to end of line */
    for ( ; ; )
    {
        if (p[0] == '\r' && p[1] == '\n' && 
            (p[2] != ' ' && p[2] != '\t') )
        {
            p[0] = 0;
            (*line) = p + 2;
            break;
        }
        p ++;
    }

    /* remove trailing spaces */
    p--;
    while (p[0] == ' ' || p[0] == '\t')
        p--;

    p[1] = 0;

    return MI_TRUE;
}


static MI_Boolean _getHeaderField(
    Http_SR_SocketData* handler,
    char ** line)
{
    /* expecting  Request-Line = Method SP Request-URI SP HTTP-Version CRLF
        Read more: http://www.faqs.org/rfcs/rfc2616.html#ixzz0jKdjJdZv
    */
    char* name = *line;
    char* value = NULL;
    int nameHashCode;

    if (!_getNameValuePair(line, &value, &nameHashCode))
        return MI_FALSE;


    if (nameHashCode == _HashCode('c','h',14) && /*Content-Length*/
        Strcasecmp(name,"Content-Length") == 0)
    {
            handler->contentLength = Strtoull(value, NULL, 10);
            /*if ( handler->contentLength > HTTP_MAX_CONTENT )
                return MI_FALSE;*/
    }
    else if (nameHashCode == _HashCode('t','g',17) && /*Transfer-Encoding*/
        Strcasecmp(name,"Transfer-Encoding") == 0)
    {
            handler->contentLength = -1;
    }

    else
    {
        if (handler->recvHeaders.sizeHeaders < MI_COUNT(handler->recvHeaderFields))
        {
            handler->recvHeaderFields[handler->recvHeaders.sizeHeaders].name = name;
            handler->recvHeaderFields[handler->recvHeaders.sizeHeaders].value = value;
            handler->recvHeaders.sizeHeaders++;
        }
        else
        {
            LOGW_CHAR(("too many http headers; skipping %s: %s\n", name, value));
        }
    }

    return MI_TRUE;
}

static MI_Boolean _getChunkSize(
    const char * line,
    MI_Uint32* chunkSize)
{
    *chunkSize = 0;

    while(*line)
    {
        char c = *line;

        if (c >= '0' && c <= '9')
            *chunkSize = *chunkSize * 16 + (c - '0');
        else if (c >= 'a' && c <= 'f')
            *chunkSize = *chunkSize * 16 + (c - 'a' + 10);
        else if (c >= 'A' && c <= 'F')
            *chunkSize = *chunkSize * 16 + (c - 'A' + 10);
        else
            break;

        line++;
    }

    return MI_TRUE;
}


static MI_Boolean _getRequestLine(
    Http_SR_SocketData* handler,
    char ** line)
{
    size_t index;
    /* expecting  Request-Line = Method SP Request-URI SP HTTP-Version CRLF
        Read more: http://www.faqs.org/rfcs/rfc2616.html#ixzz0jKdjJdZv
    */

    /* initialize header */
    handler->recvHeaders.sizeHeaders = 0;
    handler->recvHeaders.headers = handler->recvHeaderFields;

    /* find http code */
    {
        /* skip http version, that is in format
                HTTP-Version   = "HTTP" "/" 1*DIGIT "." 1*DIGIT
                http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.1

                Status-Line = HTTP-Version SP Status-Code SP Reason-Phrase CRLF
        */
        const char* s = *line + 9; /*+ HTTP/1.1<sp>*/

        handler->recvHeaders.httpError = Strtoul(s, NULL, 10);
    }

    /* skip to end of line */
    for ( index = 1; index < handler->recvievedSize; index++ )
    {
        if ((*line)[index-1] == '\r' && (*line)[index] == '\n' )
        {
            (*line) = (*line) + index + 1;
            return MI_TRUE;
        }
    }

    return MI_FALSE;
}

static MI_Result _Sock_Read(
    Http_SR_SocketData* handler,
    void* buf,
    size_t buf_size,
    size_t* sizeRead)
{
    int res;

    if (!handler->ssl)
        return Sock_Read(handler->base.sock, buf, buf_size, sizeRead);

    handler->base.mask &= ~SELECTOR_WRITE;
    handler->base.mask |= SELECTOR_READ;
    handler->reverseOperations = MI_FALSE;

    *sizeRead = 0;

    res = SSL_read(handler->ssl, buf, buf_size);
    PRINTF(("ssl read %d\n", res));

    if ( res == 0 )
        return MI_RESULT_OK;    /* connection closed */

    if ( res > 0 )
    {
        *sizeRead = res;
        return MI_RESULT_OK;    /* ok */
    }

    switch (SSL_get_error(handler->ssl, res))
    {
    case SSL_ERROR_WANT_WRITE:
        handler->reverseOperations = MI_TRUE;   /* wait until write is allowed */
        handler->base.mask &= ~SELECTOR_READ;
        handler->base.mask |= SELECTOR_WRITE;
        PRINTF(("ssl read/accept WANT_WRITE\n"));
        return MI_RESULT_WOULD_BLOCK;

    case SSL_ERROR_WANT_READ:
        PRINTF(("ssl read/accept WANT_READ\n"));
        return MI_RESULT_WOULD_BLOCK;

    case SSL_ERROR_SYSCALL:
        if (EAGAIN == errno ||
            EWOULDBLOCK == errno ||
            EINPROGRESS == errno)
            return MI_RESULT_WOULD_BLOCK;

        LOGW_CHAR(("ssl-read: unexpected sys error %d\n", errno));
        break;

    default:
        {
            /* print error */
            unsigned long err = ERR_get_error();
            while (err)
            {
                char err_txt[200];
                ERR_error_string_n(err, err_txt, sizeof(err_txt));

                LOGW_CHAR(("ssl-read error: %d [%s]\n", (int)err, err_txt));

                err = ERR_get_error();
            }
        }
        break;
    }
    return MI_RESULT_FAILED;
}

static MI_Result _Sock_Write(
    Http_SR_SocketData* handler,
    void* buf,
    size_t buf_size,
    size_t* sizeWritten)
{
    int res;

    if (!handler->ssl)
        return Sock_Write(handler->base.sock, buf, buf_size, sizeWritten);

    /* Do not clear READ flag, since 'close' notification
    delivered as READ event */
    handler->base.mask &= ~SELECTOR_READ; 
    handler->base.mask |= SELECTOR_WRITE;
    handler->reverseOperations = MI_FALSE;

    *sizeWritten = 0;

    if (handler->connectDone)
    {
        res = SSL_write(handler->ssl, buf, buf_size);
        PRINTF(("ssl write %d\n", res));
    }
    else
    {
        res = SSL_connect(handler->ssl);
        PRINTF(("ssl connect %d\n", res));
        if ( res > 0 )
        {
            /* we are done with accpet */
            handler->connectDone = MI_TRUE;
            return _Sock_Write(handler,buf,buf_size,sizeWritten);
        }
        /* perform regular error checking */
    }


    if ( res == 0 )
        return MI_RESULT_OK;    /* connection closed */

    if ( res > 0 )
    {
        *sizeWritten = res;
        return MI_RESULT_OK;    /* ok */
    }

    switch (SSL_get_error(handler->ssl, res))
    {
    case SSL_ERROR_WANT_WRITE:
        PRINTF(("ssl write/connetc WANT_WRITE\n"));
        return MI_RESULT_WOULD_BLOCK;

    case SSL_ERROR_WANT_READ:
        PRINTF(("ssl write/connetc WANT_READ\n"));
        handler->reverseOperations = MI_TRUE;   /* wait until write is allowed */
        handler->base.mask |= SELECTOR_READ;
        handler->base.mask &= ~SELECTOR_WRITE; 
        return MI_RESULT_WOULD_BLOCK;

    case SSL_ERROR_SYSCALL:
        if (EAGAIN == errno ||
            EWOULDBLOCK == errno ||
            EINPROGRESS == errno)
            return MI_RESULT_WOULD_BLOCK;

        LOGW_CHAR(("ssl-write: unexpected sys error %d\n", errno));
        break;

    default:
        break;
    }
    return MI_RESULT_FAILED;
}

static Http_CallbackResult _ReadHeader(
    Http_SR_SocketData* handler)
{
    char* buf;
    char* currentLine;
    char* data;
    size_t buf_size, received, index;
    MI_Result r;
    MI_Boolean fullHeaderReceived = MI_FALSE;

    /* are we done with header? */
    if (handler->recvingState != RECV_STATE_HEADER)
        return PRT_CONTINUE;

    buf = handler->recvBuffer + handler->recvievedSize;
    buf_size = handler->recvBufferSize - handler->recvievedSize;
    received = 0;

    r = _Sock_Read(handler, buf, buf_size, &received);
    PRINTF(("%d: read r = %d, recv = %d; reverse %d\n", (int)handler->base.sock, (int)r, (int)received, (int)handler->reverseOperations ));

    if ( r == MI_RESULT_OK && 0 == received )
        return PRT_RETURN_FALSE; /* conection closed */

    if ( r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK )
        return PRT_RETURN_FALSE;

    if (!received)
        return PRT_RETURN_TRUE;

    handler->recvievedSize += received;

    /* check header */
    PRINTF_2(("%s\n",buf));

    /* did we get full header? */
    buf = handler->recvBuffer;
    for ( index = 3; index < handler->recvievedSize; index++ )
    {
        if (buf[index-3] == '\r' && buf[index-1] == '\r' &&
            buf[index-2] == '\n' && buf[index] == '\n' )
        {
            fullHeaderReceived = MI_TRUE;
            break;
        }
    }

    if (!fullHeaderReceived )
    {
        if ( handler->recvievedSize <  handler->recvBufferSize )
            return PRT_RETURN_TRUE; /* continue reading */

        if ( handler->recvBufferSize < MAX_HEADER_SIZE )
        {
            buf = realloc(handler->recvBuffer, handler->recvBufferSize * 2);

            if (!buf)
                return PRT_RETURN_FALSE;

            handler->recvBufferSize *= 2;
            handler->recvBuffer = buf;
            return _ReadHeader(handler);
        }
        else
        {
            /* http header is too big - drop connection */
            LOGW((T("http header is too big; dropping connection\n")));
            return PRT_RETURN_FALSE;
        }
    }

    /* consume data */
    currentLine = buf;
    data = buf + index + 1; /* pointer to data in case we got some */

    if (!_getRequestLine(handler, &currentLine))
        return PRT_RETURN_FALSE;


    while ((data-currentLine) > 3)
    {
        if (!_getHeaderField(handler, &currentLine))
            return PRT_RETURN_FALSE;

    }

    /* Check if we have to deal with chunked-encoded data */
    if (handler->contentLength < 0)
    {
        handler->recvievedSize -= index + 1;

        /* Invoke user's callback with header information */
        {
            HttpClient* self = (HttpClient*)handler->base.data;

            if (!(*self->callbackOnResponse)( self, self->callbackData, &handler->recvHeaders, 
                handler->contentLength, handler->contentLength == 0, 0))
                return PRT_RETURN_FALSE;
        }

        /* remove consumed header part */
        memmove(handler->recvBuffer, data, handler->recvievedSize);

        handler->recvingState = RECV_STATE_CHUNKHEADER;
        return PRT_CONTINUE;
    }

    /* Allocate zero-terminated buffer */
    handler->recvPage = (Page*)malloc(sizeof(Page) + (size_t)handler->contentLength + 1);

    if (!handler->recvPage)
        return PRT_RETURN_FALSE;

    ((char*)(handler->recvPage + 1))[handler->contentLength] = 0;

    handler->recvPage->u.s.size = (unsigned int)handler->contentLength;
    handler->recvPage->u.s.next = 0;

    handler->recvievedSize -= index + 1;

    /* Verify that we have not more than 'content-length' bytes in buffer left 
        If we have more, assuming http client is invalid and drop connection */
    if (handler->recvievedSize > (size_t)handler->contentLength)
    {
        LOGW((T("http payload is bigger than content-length\n")));
        return PRT_RETURN_FALSE;
    }

    memcpy( handler->recvPage + 1, data, handler->recvievedSize );
    handler->recvingState = RECV_STATE_CONTENT;

    PRINTF_2(("full header read; page size %d, position %d\n", handler->recvPage->u.s.size, handler->recvievedSize));

    /* Invoke user's callback with header information */
    {
        HttpClient* self = (HttpClient*)handler->base.data;

        if (!(*self->callbackOnResponse)( self, self->callbackData, &handler->recvHeaders, 
            handler->contentLength, handler->contentLength == 0, 0))
            return PRT_RETURN_FALSE;
    }

    return PRT_CONTINUE;
}

static Http_CallbackResult _ReadData(
    Http_SR_SocketData* handler)
{
    //HttpClient* self = (HttpClient*)handler->base.data;
    char* buf;
    size_t buf_size, received;
    MI_Result r;

    /* are we in the right state? */
    if (handler->recvingState != RECV_STATE_CONTENT)
        return PRT_RETURN_FALSE;

    buf = ((char*)(handler->recvPage + 1)) + handler->recvievedSize;
    buf_size = (size_t)(handler->contentLength - handler->recvievedSize);
    received = 0;

    if (buf_size)
    {
        r = _Sock_Read(handler, buf, buf_size, &received);

        PRINTF(("%d: read r = %d, recv = %d\n", (int)handler->base.sock, (int)r, (int)received ));

        if ( r == MI_RESULT_OK && 0 == received )
            return PRT_RETURN_FALSE; /* conection closed */

        if ( r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK )
            return PRT_RETURN_FALSE;

        handler->recvievedSize += received;
    }

    /* did we get all data? */
    PRINTF_2(("dt status - %d / %d\n", (int)handler->recvievedSize, (int)handler->contentLength));

    if ( handler->recvievedSize != (size_t)handler->contentLength )
        return PRT_RETURN_TRUE;
 
    /* Invoke user's callback with header information */
    {
        HttpClient* self = (HttpClient*)handler->base.data;

        if (!(*self->callbackOnResponse)( self, self->callbackData, 0, 
            handler->contentLength, MI_TRUE, &handler->recvPage))
            return PRT_RETURN_FALSE;

        /* status callback */
        handler->status = MI_RESULT_OK;
        (*self->callbackOnStatus)(
            self, 
            self->callbackData, 
            MI_RESULT_OK );
    }


    if (handler->recvPage)
        free(handler->recvPage);

    handler->recvPage = 0;
    handler->recvievedSize = 0;
    memset(&handler->recvHeaders, 0, sizeof(handler->recvHeaders));
    handler->recvingState = RECV_STATE_HEADER;
    return PRT_CONTINUE;
}


static Http_CallbackResult _ReadChunkHeader(
    Http_SR_SocketData* handler)
{
    char* buf;
    char* currentLine;
    char* data;
    size_t buf_size, received, index;
    MI_Result r;
    MI_Boolean fullHeaderReceived = MI_FALSE;
    MI_Uint32   chunkSize = 0;
    MI_Boolean connectionClosed = MI_FALSE;

    /* are we done with header? */
    if (handler->recvingState != RECV_STATE_CHUNKHEADER)
        return PRT_CONTINUE;

    buf = handler->recvBuffer + handler->recvievedSize;
    buf_size = handler->recvBufferSize - handler->recvievedSize;
    received = 0;

    r = _Sock_Read(handler, buf, buf_size, &received);
    PRINTF(("%d: read r = %d, recv = %d; reverse %d\n", (int)handler->base.sock, (int)r, (int)received, (int)handler->reverseOperations ));

    if ( r == MI_RESULT_OK && 0 == received )
    {
        if (!handler->recvBufferSize)
            return PRT_RETURN_FALSE; /* conection closed */

        connectionClosed = MI_TRUE;
    }

    if ( r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK )
        return PRT_RETURN_FALSE;

    if (!received && !handler->recvBufferSize)
        return PRT_RETURN_TRUE;

    handler->recvievedSize += received;

    /* check header */
    PRINTF_2(("%s\n",buf));

    /* did we get full header? */
    buf = handler->recvBuffer;
    for ( index = 1; index < handler->recvievedSize; index++ )
    {
        if (buf[index-1] == '\r' && buf[index] == '\n' )
        {
            fullHeaderReceived = MI_TRUE;
            break;
        }
    }

    if (!fullHeaderReceived )
    {
        if (connectionClosed)
            return PRT_RETURN_FALSE; /* conection closed */

        if ( handler->recvievedSize <  handler->recvBufferSize )
            return PRT_RETURN_TRUE; /* continue reading */

        if ( handler->recvBufferSize < MAX_HEADER_SIZE )
        {
            buf = realloc(handler->recvBuffer, handler->recvBufferSize * 2);

            if (!buf)
                return PRT_RETURN_FALSE;

            handler->recvBufferSize *= 2;
            handler->recvBuffer = buf;
            return _ReadChunkHeader(handler);
        }
        else
        {
            /* http chunk header is too big - drop connection */
            LOGW((T("http chunk header is too big; dropping connection\n")));
            return PRT_RETURN_FALSE;
        }
    }

    /* consume data */
    currentLine = buf;
    data = buf + index + 1; /* pointer to data in case we got some */

    if (!_getChunkSize(currentLine, &chunkSize))
        return PRT_RETURN_FALSE;

    if (0 == chunkSize)
    {
        /* last chunk received */

        /* Invoke user's callback with header information */
        {
            HttpClient* self = (HttpClient*)handler->base.data;

            if (!(*self->callbackOnResponse)( self, self->callbackData, 0, 
                handler->contentLength, MI_TRUE, 0))
                return PRT_RETURN_FALSE;

            /* status callback */
            handler->status = MI_RESULT_OK;
            (*self->callbackOnStatus)(
                self, 
                self->callbackData, 
                MI_RESULT_OK );
        }

        /* clean up state */
        handler->recvPage = 0;
        handler->recvievedSize = 0;
        memset(&handler->recvHeaders, 0, sizeof(handler->recvHeaders));
        handler->recvingState = RECV_STATE_HEADER;

        if (connectionClosed)
            return PRT_RETURN_FALSE; /* conection closed */

        return PRT_CONTINUE;
    }

    /* Allocate zero-terminated buffer */
    handler->recvPage = (Page*)malloc(sizeof(Page) + (size_t)chunkSize + 2 /*CR-LF*/ + 1 /* \0 */);

    if (!handler->recvPage)
        return PRT_RETURN_FALSE;

    ((char*)(handler->recvPage + 1))[chunkSize+2] = 0;

    handler->recvPage->u.s.size = (unsigned int)chunkSize;
    handler->recvPage->u.s.next = 0;

    /* subtract header size */
    handler->recvievedSize -= index + 1;

    /* in case of small chunks we may receive more than one chunk already */
    if (handler->recvievedSize > (size_t)(chunkSize+2))
    {
        /* copy page size to page */
        memcpy( handler->recvPage + 1, data, chunkSize+2 );

        /* notify user */
        {
            HttpClient* self = (HttpClient*)handler->base.data;

            if (!(*self->callbackOnResponse)( self, self->callbackData, 0, 
                handler->contentLength, MI_FALSE, &handler->recvPage))
                return PRT_RETURN_FALSE;

            if (handler->recvPage)
                free(handler->recvPage);

            handler->recvPage = 0;
        }

        /* remove consumed part */
        memmove(handler->recvBuffer, data + chunkSize+2, handler->recvievedSize - (chunkSize+2));
        handler->recvievedSize -= (chunkSize+2);

        /* consume next chunk */
        return _ReadChunkHeader(handler);
    }

    memcpy( handler->recvPage + 1, data, handler->recvievedSize );
    handler->recvingState = RECV_STATE_CHUNKDATA;

    if (connectionClosed)
        return PRT_RETURN_FALSE; /* conection closed */

    return PRT_CONTINUE;
}
static Http_CallbackResult _ReadChunkData(
    Http_SR_SocketData* handler)
{
    //HttpClient* self = (HttpClient*)handler->base.data;
    char* buf;
    size_t buf_size, received;
    MI_Result r;

    /* are we in the right state? */
    if (handler->recvingState != RECV_STATE_CHUNKDATA)
        return PRT_RETURN_FALSE;

    buf = ((char*)(handler->recvPage + 1)) + handler->recvievedSize;
    buf_size = (size_t)(handler->recvPage->u.s.size + 2 /* CR-LF */ - handler->recvievedSize);
    received = 0;

    if (buf_size)
    {
        r = _Sock_Read(handler, buf, buf_size, &received);

        PRINTF(("%d: read r = %d, recv = %d\n", (int)handler->base.sock, (int)r, (int)received ));

        if ( r == MI_RESULT_OK && 0 == received )
            return PRT_RETURN_FALSE; /* conection closed */

        if ( r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK )
            return PRT_RETURN_FALSE;

        handler->recvievedSize += received;
    }

    /* did we get all data? */
    PRINTF_2(("dt status - %d / %d\n", (int)handler->recvievedSize, (int)handler->contentLength));

    if ( handler->recvievedSize != (size_t)(handler->recvPage->u.s.size + 2 /* CR-LF */) )
        return PRT_RETURN_TRUE;
 
    /* Invoke user's callback with header information */
    {
        HttpClient* self = (HttpClient*)handler->base.data;

        if (!(*self->callbackOnResponse)( self, self->callbackData, 0, 
            handler->contentLength, MI_FALSE, &handler->recvPage))
            return PRT_RETURN_FALSE;

    }


    if (handler->recvPage)
        free(handler->recvPage);

    handler->recvPage = 0;
    handler->recvievedSize = 0;
    memset(&handler->recvHeaders, 0, sizeof(handler->recvHeaders));
    handler->recvingState = RECV_STATE_CHUNKHEADER;
    return PRT_CONTINUE;
}


static Http_CallbackResult _WriteHeader(
    Http_SR_SocketData* handler)
{
    char* buf;
    size_t buf_size, sent;
    MI_Result r;

    /* Do we have any data to send? */
    if (!handler->sendHeader)
        return PRT_RETURN_TRUE;

    /* are we done with header? */
    if (handler->sendingState == RECV_STATE_CONTENT)
        return PRT_CONTINUE;

    buf = ((char*)(handler->sendHeader + 1)) + handler->sentSize;
    buf_size = handler->sendHeader->u.s.size - handler->sentSize;
    sent = 0;

    r = _Sock_Write(handler, buf, buf_size, &sent);

    PRINTF(("%d: write r = %d, sent = %d\n", (int)handler->base.sock, (int)r, (int)sent ));

    if ( r == MI_RESULT_OK && 0 == sent )
        return PRT_RETURN_FALSE; /* conection closed */

    if ( r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK )
        return PRT_RETURN_FALSE;

    handler->sentSize += sent;

    /* did we get all data? */

    if ( handler->sentSize != handler->sendHeader->u.s.size )
        return PRT_RETURN_TRUE;
 
    free(handler->sendHeader);
    handler->sendHeader = 0;
    handler->sentSize = 0;
    handler->sendingState = RECV_STATE_CONTENT;

    return PRT_CONTINUE;
}

static Http_CallbackResult _WriteData(
    Http_SR_SocketData* handler)
{
    char* buf;
    size_t buf_size, sent;
    MI_Result r;

    /* are we in the right state? */
    if (handler->sendingState != RECV_STATE_CONTENT)
        return PRT_RETURN_FALSE;

    if (!handler->sendPage)
    {   /* no content*/
        handler->sentSize = 0;
        handler->sendingState = RECV_STATE_HEADER;
        handler->base.mask &= ~SELECTOR_WRITE;
        handler->base.mask |= SELECTOR_READ;

        return PRT_CONTINUE;
    }

    buf = ((char*)(handler->sendPage + 1)) + handler->sentSize;
    buf_size = handler->sendPage->u.s.size - handler->sentSize;
    sent = 0;

    r = _Sock_Write(handler, buf, buf_size, &sent);

    PRINTF(("%d: write r = %d, sent = %d\n", (int)handler->base.sock, (int)r, (int)sent ));

    if ( r == MI_RESULT_OK && 0 == sent )
        return PRT_RETURN_FALSE; /* conection closed */

    if ( r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK )
        return PRT_RETURN_FALSE;

    handler->sentSize += sent;

    /* did we get all data? */

    if ( handler->sentSize != handler->sendPage->u.s.size )
        return PRT_RETURN_TRUE;
 
    free(handler->sendPage);
    handler->sendPage = 0;
    handler->sentSize = 0;
    handler->sendingState = RECV_STATE_HEADER;
    handler->base.mask &= ~SELECTOR_WRITE;
    handler->base.mask |= SELECTOR_READ;

    return PRT_CONTINUE;
}


static MI_Boolean _RequestCallbackRead(
    Http_SR_SocketData* handler)
{
    switch (_ReadHeader(handler))
    {
    case PRT_CONTINUE: break;
    case PRT_RETURN_TRUE: return MI_TRUE;
    case PRT_RETURN_FALSE: return MI_FALSE;
    }

    if (handler->recvingState == RECV_STATE_CONTENT)
    {
        switch (_ReadData(handler))
        {
        case PRT_CONTINUE: break;
        case PRT_RETURN_TRUE: return MI_TRUE;
        case PRT_RETURN_FALSE: return MI_FALSE;
        }
    }

    if (handler->recvingState == RECV_STATE_CHUNKHEADER)
    {
        switch (_ReadChunkHeader(handler))
        {
        case PRT_CONTINUE: break;
        case PRT_RETURN_TRUE: return MI_TRUE;
        case PRT_RETURN_FALSE: return MI_FALSE;
        }
    }
    if (handler->recvingState == RECV_STATE_CHUNKDATA)
    {
        switch (_ReadChunkData(handler))
        {
        case PRT_CONTINUE: break;
        case PRT_RETURN_TRUE: return MI_TRUE;
        case PRT_RETURN_FALSE: return MI_FALSE;
        }
    }

    return MI_TRUE;
}

static MI_Boolean _RequestCallbackWrite(
    Http_SR_SocketData* handler)
{
    switch (_WriteHeader(handler))
    {
    case PRT_CONTINUE: break;
    case PRT_RETURN_TRUE: return MI_TRUE;
    case PRT_RETURN_FALSE: return MI_FALSE;
    }

    switch (_WriteData(handler))
    {
    case PRT_CONTINUE: break;
    case PRT_RETURN_TRUE: return MI_TRUE;
    case PRT_RETURN_FALSE: return MI_FALSE;
    }
    return MI_TRUE;
}


static MI_Boolean _RequestCallback(
    Selector* sel,
    Handler* handlerIn,
    MI_Uint32 mask, 
    MI_Uint64 currentTimeUsec)
{
    Http_SR_SocketData* handler = (Http_SR_SocketData*)handlerIn;
    sel=sel;

    if ( ((mask & SELECTOR_READ) != 0 && !handler->reverseOperations) ||
        ((mask & SELECTOR_WRITE) != 0 && handler->reverseOperations) )
    {
        if (!_RequestCallbackRead(handler))
            return MI_FALSE;
    }

    if ( ((mask & SELECTOR_WRITE) != 0 && !handler->reverseOperations) ||
        ((mask & SELECTOR_READ) != 0 && handler->reverseOperations) )
    {
        if (!_RequestCallbackWrite(handler))
            return MI_FALSE;
    }
    
    /* re-set timeout - if we performed R/W operation, set timeout depending where we are in communication */
    if (mask & (SELECTOR_READ | SELECTOR_WRITE))
    {
        handler->base.fireTimeoutAt = currentTimeUsec + handler->timeoutUsec;
    }

    /* Close conenction by timeout */
    if (mask & SELECTOR_TIMEOUT)
    {
        if (MI_RESULT_OK != handler->status)
            handler->status = MI_RESULT_TIME_OUT;

        return MI_FALSE;
    }

    if ((mask & SELECTOR_REMOVE) != 0 ||
        (mask & SELECTOR_DESTROY) != 0)
    {
        HttpClient* self = (HttpClient*)handler->base.data;

        /* notify next stack layer */
        if (MI_RESULT_OK != handler->status)
            (*self->callbackOnStatus)(
                self, 
                self->callbackData, 
                handler->status);

        self->connector = 0;

        if (handler->ssl)
            SSL_free(handler->ssl);

        PRINTF(("%d: close\n", (int)handler->base.sock));

        Sock_Close(handler->base.sock);

        if (handler->recvPage)
            free(handler->recvPage);

        if (handler->sendPage)
            free(handler->sendPage);

        if (handler->sendHeader)
            free(handler->sendHeader);

        free(handler->recvBuffer);
        free(handler);
    }

    return MI_TRUE;
}

#ifdef CONFIG_POSIX
static MI_Result _CreateSSLContext(HttpClient* self)
{
    SSL_CTX * sslContext = 0;

    sslContext = SSL_CTX_new(SSLv23_method());

    if (!sslContext)
    {
        LOGE_CHAR((
            "---> SSL: cannot create ssl context"));
        return MI_RESULT_FAILED;
    }
    SSL_CTX_set_quiet_shutdown(sslContext, 1);
    SSL_CTX_set_mode(sslContext, SSL_MODE_AUTO_RETRY);
    SSL_CTX_set_mode(sslContext, SSL_MODE_ENABLE_PARTIAL_WRITE);
    SSL_CTX_set_session_cache_mode(sslContext, SSL_SESS_CACHE_OFF);
#if 0
    /* Check if there is a certificate file (file containing server
    ** certificate) specified. If specified, validate and load the
    ** certificate.
    */
    {
        /* load the specified server certificates */
        LOGI_CHAR(("---> SSL: Loading server certificate from: %s", 
            GetPath(ID_PEMFILE)));

        if (SSL_CTX_use_certificate_file(sslContext,
            GetPath(ID_PEMFILE), SSL_FILETYPE_PEM) <=0)
        {
            LOGE_CHAR(("---> SSL: No server certificate found in %s", 
                GetPath(ID_PEMFILE)));
            SSL_CTX_free(sslContext);
            return MI_RESULT_FAILED;
        }
    }

    /*
    ** Check if there is a key file (file containing server
    ** private key) specified and the key was not already loaded.
    ** If specified, validate and load the key.
    */
    {
        /* load the specified server certificates */
        LOGI_CHAR(("---> SSL: Loading certificate's private key from: %s",
            GetPath(ID_KEYFILE)));

        //
        // load given private key and check for validity
        //
        if (!_verifyPrivateKey(sslContext, GetPath(ID_KEYFILE)))
        {
            LOGE_CHAR((
                "---> SSL: No server certificate found in %s",
                GetPath(ID_KEYFILE)));
            SSL_CTX_free(sslContext);
            return MI_RESULT_FAILED;
        }
    }
#endif
    self->sslContext = sslContext;
    return MI_RESULT_OK;
}
#endif

static MI_Result _CreateConnectorSocket(
    HttpClient* self,
    const char* host,
    unsigned short port,
    MI_Boolean secure)
{
    Addr addr;
    MI_Result r;
    Sock s;
    Http_SR_SocketData* h;
    MI_Uint64 currentTimeUsec;

    /* timeout calculation */
    if (MI_RESULT_OK != Time_Now(&currentTimeUsec))
        return MI_RESULT_FAILED;

    // Initialize address.
    r = Addr_Init(&addr, host, port);
    if (r != MI_RESULT_OK)
        return MI_RESULT_FAILED;

    // Create client socket.
    r = Sock_Create(&s);
    if (r != MI_RESULT_OK)
    {
        Sock_Close(s);
        return MI_RESULT_FAILED;
    }

    r = Sock_SetBlocking(s, MI_FALSE);
    if (r != MI_RESULT_OK)
    {
        Sock_Close(s);
        return MI_RESULT_FAILED;
    }

    // Connect to the server.
    r = Sock_Connect(s, &addr);
    if (r != MI_RESULT_OK && r != MI_RESULT_WOULD_BLOCK)
    {
        Sock_Close(s);
        return MI_RESULT_FAILED;
    }

    /* Create handler */
    h = (Http_SR_SocketData*)calloc(1, sizeof(Http_SR_SocketData));

    if (!h)
    {
        Sock_Close(s);
        return MI_RESULT_FAILED;
    }

    h->recvBufferSize = INITIAL_BUFFER_SIZE;
    h->recvBuffer = (char*)calloc(1, h->recvBufferSize);
    if (!h->recvBuffer)
    {
        free(h);
        Sock_Close(s);
        return MI_RESULT_FAILED;
    }

    h->base.sock = s;
    h->base.mask = SELECTOR_EXCEPTION;
    h->base.callback = _RequestCallback;
    h->base.data = self;
    h->timeoutUsec = DEFAULT_HTTP_TIMEOUT_USEC;
    h->base.fireTimeoutAt = currentTimeUsec + h->timeoutUsec;

    /* ssl support */
    if (secure)
    {
        h->ssl = SSL_new(self->sslContext);

        if (!h->ssl)
        {
            LOGW((T("ssl_new() failed\n")));
            free(h);
            Sock_Close(s);
            return MI_RESULT_FAILED;
        }

        if (!(SSL_set_fd(h->ssl, s) ))
        {
            LOGW((T("ssl_set_fd() failed\n")));
            SSL_free(h->ssl);
            free(h);
            Sock_Close(s);
            return MI_RESULT_FAILED;
        }

        SSL_set_connect_state(h->ssl);
    }

    /* Watch for read events on the incoming connection */
    r = Selector_AddHandler(self->selector, &h->base);

    if (r != MI_RESULT_OK)
    {
        LOGW((T("Selector_AddHandler() failed\n")));
        if (secure)
            SSL_free(h->ssl);
        free(h);
        Sock_Close(s);
        return MI_RESULT_FAILED;
    }

    self->connector = h;

    return MI_RESULT_OK;
}


static MI_Result _New_Http(
    HttpClient** selfOut,
    Selector* selector, /*optional, maybe NULL*/
    HttpClientCallbackOnStatus statusCallback,
    HttpClientCallbackOnResponse  responseCallback,
    void* callbackData)
{
    HttpClient* self;

    /* Check parameters */
    if (!selfOut)
        return MI_RESULT_INVALID_PARAMETER;

    /* Clear output parameter */
    *selfOut = NULL;

    /* Allocate structure */
    {
        self = (HttpClient*)calloc(1, sizeof(HttpClient));

        if (!self)
            return MI_RESULT_FAILED;
    }

    if (selector)
    {   /* attach the exisiting selector */
        self->selector = selector;
        self->internalSelectorUsed = MI_FALSE;
    }
    else
    {   /* creaet a new selector */
        /* Initialize the network */
        Sock_Start();

        /* Initialize the selector */
        if (Selector_Init(&self->internalSelector) != MI_RESULT_OK)
        {
            free(self);
            return MI_RESULT_FAILED;
        }
        self->selector = &self->internalSelector;
        self->internalSelectorUsed = MI_TRUE;
    }

    /* Save the callback and callbackData */
    self->callbackOnResponse = responseCallback;
    self->callbackOnStatus = statusCallback;
    self->callbackData = callbackData;

    /* Set the magic number */
    self->magic = _MAGIC;

    /* Set output parameter */
    *selfOut = self;
    return MI_RESULT_OK;
}



static size_t _GetHeadersSize(
    const HttpClientRequestHeaders* headers)
{
    size_t res = 0;
    size_t index = 0;

    while (index < headers->size)
    {
        res += Strlen(headers->data[index]);
        res += 2; /* \r \n pair */
        index++;
    }

    return res;
}

static Page* _CreateHttpHeader(
    const char* verb,
    const char* uri,
    const HttpClientRequestHeaders* headers,
    size_t size)
{
    Page* page = 0;
    size_t pageSize = 0;
    int r;
    char* p;

#define HTTP_HEADER_FORMAT "%s %s HTTP/1.1\r\n" \
    "Content-Length: %d\r\n"\
    "Connection: Keep-Alive\r\n" \
    "Host: host\r\n"

#define HTTP_HEADER_FORMAT_NOCL "%s %s HTTP/1.1\r\n" \
    "Connection: Keep-Alive\r\n" \
    "Host: host\r\n"

    /* calculate approximate page size */
    if (!verb)
        verb = "POST";
    pageSize += sizeof(HTTP_HEADER_FORMAT) + 10; /* format + 10 digits of content length */
    pageSize += Strlen(verb);
    pageSize += Strlen(uri);

    if (headers)
        pageSize += _GetHeadersSize(headers);

    page = (Page*)malloc(pageSize + sizeof(Page));

    if (!page)
        return 0;

    /* clear header */
    memset(page, 0, sizeof(Page));

    p = (char*)(page + 1);

    if (size)
        r = Snprintf(p, pageSize, HTTP_HEADER_FORMAT, verb, uri, (int)size);
    else
        r = Snprintf(p, pageSize, HTTP_HEADER_FORMAT_NOCL, verb, uri);

    if (r < 0)
    {
        free(page);
        return 0;
    }

    p += r;
    pageSize -= r;

    if (headers)
    {
        size_t index = 0;

        while (index < headers->size)
        {
            r = (int)Strlcpy(p,headers->data[index], pageSize);
            p += r;
            pageSize -= r;
            r = (int)Strlcpy(p,"\r\n", pageSize);
            p += r;
            pageSize -= r;

            index++;
        }
    }

    /* add trailing \r\n */
    r = (int)Strlcpy(p,"\r\n", pageSize);
    p += r;
    pageSize -= r;

    page->u.s.size = (unsigned int)(p - (char*)(page+1));

    return page;
}

/* ************************************************************************* *\
                                HTTP CLIENT
\* ************************************************************************* */
/*
    Creates new http client.

    Parameters:
    selfOut - [out] newly created http object (or null if failed).
    selector - [opt] selector to use for socket monitoring. If selector not specified,
            private one is created.
    host - host address
    port - port number
    secure - flag that indicates if http or https conneciton is required

    Returns:
    'OK' on success or error code otherwise
*/
MI_Result HttpClient_New_Connector(
    HttpClient** selfOut,
    Selector* selector, /*optional, maybe NULL*/
    const char* host,
    unsigned short port,
    MI_Boolean secure,
    HttpClientCallbackOnStatus statusCallback,
    HttpClientCallbackOnResponse  responseCallback,
    void* callbackData)
{
    HttpClient* self;
    MI_Result r;

    /* allocate this, inits selector */
    r = _New_Http(selfOut, selector, statusCallback, 
        responseCallback, callbackData);

    if (MI_RESULT_OK != r)
        return r;

    self = *selfOut;

#ifdef CONFIG_POSIX
    /* Allocate ssl context */
    if (secure)
    {
        /* init ssl */
        SSL_library_init();

        /* create context */
        r = _CreateSSLContext(self);

        if (r != MI_RESULT_OK)
        {
            HttpClient_Delete(self);
            return r;
        }
    }
#else
    MI_UNUSED(secure);
#endif

    /* Create http connector socket */
    {
        r = _CreateConnectorSocket(self, host, port, secure);

        if (r != MI_RESULT_OK)
        {
            HttpClient_Delete(self);
            return r;
        }
    }


    return MI_RESULT_OK;
}

/*
    Deletes http object, disconnects form the server
    and frees all related resources.
    
    Parameters:
    self - http object

    Returns:
    OK
*/
MI_Result HttpClient_Delete(
    HttpClient* self)
{
    /* Check parameters */
    if (!self)
        return MI_RESULT_INVALID_PARAMETER;

    /* Check magic number */
    if (self->magic != _MAGIC)
        return MI_RESULT_INVALID_PARAMETER;

    if (self->internalSelectorUsed)
    {
        /* Release selector;
        Note: selector-destory closes all sockects in a list including connector and listener */
        Selector_Destroy(self->selector);

        /* Shutdown the network */
        Sock_Stop();
    }
    else
    {
        /* remove connector from handler */
        if (self->connector)
            Selector_RemoveHandler(self->selector, &self->connector->base);
    }

    if (self->sslContext)
        SSL_CTX_free(self->sslContext);

    /* Clear magic number */
    self->magic = 0xDDDDDDDD;

    /* Free self pointer */
    free(self);

    return MI_RESULT_OK;
}


/* 
    Sends http request.

    Parameters:
    self - http object
    uri - request's URI
    headers - [opt] extra headers for request. 
    data - [opt] content to send. if message is accepted to be sent, 
        on return *data == null (taking memory ownership)

    Returns:
    OK or appropriate error
 */
MI_Result HttpClient_StartRequest(
    HttpClient* self,
    const char* verb,
    const char* uri,
    const HttpClientRequestHeaders* headers,
    Page** data)
{

    /* check params */
    if (!self || !uri)
        return MI_RESULT_INVALID_PARAMETER;

    if (self->magic != _MAGIC)
    {
        LOGW((T("start-request: invalid magic!") ));
        return MI_RESULT_INVALID_PARAMETER;
    }

    if (!self->connector)
    {
        LOGW((T("start-request: connection was closed") ));
        return MI_RESULT_FAILED;
    }

    /* create header page */
    self->connector->sendHeader =
        _CreateHttpHeader(verb, uri, headers, (data && *data) ? (*data)->u.s.size : 0);
    
    if (data)
    {
        self->connector->sendPage = *data;
        *data = 0;
    }
    else
    {
        self->connector->sendPage = 0;
    }

    /* set status to failed, until we know more details */
    self->connector->status = MI_RESULT_FAILED;
    self->connector->sentSize = 0;
    self->connector->sendingState = RECV_STATE_HEADER;
    self->connector->base.mask |= SELECTOR_WRITE;

    _RequestCallbackWrite(self->connector);

    return MI_RESULT_OK;
}


MI_Result HttpClient_Run(
    HttpClient* self,
    MI_Uint64 timeoutUsec)
{
    /* Run the selector */
    return Selector_Run(self->selector, timeoutUsec);
}

MI_Result HttpClient_SetTimeout(
    HttpClient* self,
    MI_Uint64 timeoutUsec)
{
    MI_Uint64 currentTimeUsec = 0;

    Time_Now(&currentTimeUsec);

    /* check params */
    if (!self)
        return MI_RESULT_INVALID_PARAMETER;

    if (self->magic != _MAGIC)
    {
        LOGW((T("setTimeout: invalid magic!") ));
        return MI_RESULT_INVALID_PARAMETER;
    }

    if (!self->connector)
        return MI_RESULT_INVALID_PARAMETER;

    /* create header page */
    self->connector->timeoutUsec = timeoutUsec;
    self->connector->base.fireTimeoutAt = currentTimeUsec + self->connector->timeoutUsec;

    return MI_RESULT_OK;
}