src/vsl_api.c

Go to the documentation of this file.

/*
 * Copyright 2006, Internet2
 * Legal conditions are in file LICENSE
 * (MD5 = c434f2e53b8089d8b4d0172c7ce07360).
 */

/**
 * @file   vsl_api.c
 * @author Nikolaus Rath
 * @brief  Implements the VSL api calls
 *
 * \todo Document consequences of errors: can the function be
 * recalled? Is the socket still usable?
 */

#include "vsl.h"
#include "vfer.h"
#include "vsl_api.h"
#include "vsl_util.h"
#include "vsl_ssh.h"
#include "../poly1305aes/poly1305aes.h"
#include <sys/types.h>
#include <signal.h>
#include <sys/wait.h>

/**
 * Initializes the VFER security layer. Has to be called before any
 * other VSL functions.
 * \return
 * - 0 on success
 **/
int vsl_init(void) {
    return 0;
}

/**
 * Cleanup the ressources used by the security layer. Has to be called
 * before the program ends. In case of forked children, every process
 * has to call the function before exiting.
 * \return
 * - 0 on success
 **/
int vsl_uninit(void) {
    return 0;
}

/**
 * Establishes a secure connection to the specified host using SSL
 * Starting Mode. This function has to be called by the client. It
 * connects to the remote host via ssh and starts the server. See
 * specification for details.
 *
 * The function requires an initialized VSL socket.
 *
 * For non blocking sockets, this function might return
 * #VFER_INPROGRESS. That means that the next authentikation step
 * would block. In this case the caller has to call the function
 * repeatedly until it returns 0 or an error. vsl_select() can be
 * used to check whether a socket is ready for the next step.
 *
 * \note The SIGPIPE signal handler is set to ignore by this function.
 * If neccessary, the caller has to save the original handler and restore it
 * after authentication.
 *
 * @param[out] socket VSL socket to use
 * @param[in] host hostname to connect to
 * @param[in] user username for SSH connection, \c NULL to use current user
 * @param[in] cmd  program name of the server application
 * @param[in] argc  number of arguments in \a argv that are passed to the server application
 * @param[in] argv  arguments that is passed to the server application
 * \param[out] errmsg if the server returns an error message it is
 * stored here. Must be achar array holding at
 * least 512 bytes.
 *
 * \return
 *  - 0 on success
 *  - #VFER_REFUSED vfer connection attempt refused
 *  - #VFER_TIMEOUT vfer connection attempt timedout
 *  - #VFER_INPROGRESS if the socket is nonblocking and the call would
 *    block.
 *  - #VSL_BADPROT in case of a protocol error
 *  - #VSL_SSHF if the ssh process failed
 *  - #VSL_TEMP if the server signaled a temporary error
 *  - #VSL_PERM if the server signaled a permanent error
 *  - #VSL_ERRNO if a system call failed (errno is set accordingly)
 **/
int vsl_connect_ssh (vsl_sock* socket, const char* host, const char* user,
                     const char* cmd, int argc, const char* const *argv,
                     char* errmsg) {
    unsigned char secret[VSL_KEYLEN];
    int port, ret;

    LOG("trying ssh startup as client");

    switch(CONSTAT(socket->status)) {
    case(C_INITIALIZED):
        /* Fork SSH */
        TRY(ssh_fork(socket, host, user, cmd, argc, argv));

        /* Generate shared secret */
        TRY(gen_key(secret));

        // Update status
        socket->status = C_SSH_FORKED | C_WAIT_FD_W;

        // Testing mode
        if ( socket->test_mode &&
             !is_blocking(socket) )
            LOGFAIL(VFER_INPROGRESS, "returning, testing mode.");

    case(C_SSH_FORKED):
        // Write shared secret
        if ( (ret=ssh_send_secret(socket, secret)) != 0) {
            int bak = errno;
            ssh_close(socket, errmsg);
            errno = bak;
            return ret;
        }

        // Update status
        socket->status = C_SSH_WROTE_SEC | C_WAIT_FD_R;

        // Testing mode
        if ( socket->test_mode &&
             !is_blocking(socket) )
            LOGFAIL(VFER_INPROGRESS, "returning, testing mode.");

    case(C_SSH_WROTE_SEC):
        /* Read port nr. */
        if ( (ret=ssh_read_port(socket, &port)) != 0) {
            int bak = errno;
            ssh_close(socket, errmsg);
            errno = bak;
            return ret;
        }

        // Establish vfer connection in background
        TRY(ssh_vfer_connect(socket, host, port));
        socket->status = C_SSH_VF_CON | C_WAIT_PID;

        // Testing mode
        if ( socket->test_mode &&
             !is_blocking(socket) )
            LOGFAIL(VFER_INPROGRESS, "returning, testing mode.");

    case C_SSH_VF_CON:
        // Close ssh connection
        TRY(ssh_close(socket, errmsg));

        //! \todo reactivate this when vfer debugging is done
        /*
        // wait for vfer_connect
        LOG("waiting for vfer_connect...");
        int ret;
        if ( (ret = vfer_selectmark(socket->vfd, VFER_READABLE | VFER_WRITABLE
                                    | VFER_EXCEPTION)) != 0)
            LOGFATAL("vfer_selectmark failed: %s!", vfer_errortext(ret));
        if ( (ret = vfer_select(1, &(socket->vfd), NULL)) != 1 )
            LOGFATAL("vfer_select failed: %s!", vfer_errortext(ret));
        if ( vfer_selecttest(socket->vfd) & VFER_EXCEPTION ) {
            if (  (ret = vfer_sockerror(socket->vfd)) == VFER_TIMEOUT ||
                  ret == VFER_REFUSED)
                LOGFAIL(ret, "vfer_connect failed: %s", vfer_errortext(ret));
            LOGFATAL("vfer_connect failed: %s", vfer_errortext(ret));
        }
        */

        /* Generate keys */
        init_keys_c(socket, secret);

        socket->status = C_ESTABLISHED;
        socket->auth = VSL_AUTH_SSH;
        break;

    default:
        /* Invalid state */
        LOGFAIL(VFER_BADSOCK, "invalid socket state");
    }

    LOG("ssh startup successful.");
    return 0;
}

/**
 * Establishes a secure connection to the specified host using SSL
 * Starting Mode. This function has to be called by the server. It
 * reads the secret from stdin, listens on a free port, prints the
 * port nr. to stdout, accepts a connection and daemonizes. See
 * specification for details.
 *
 * This call always blocks.
 *
 * The function requires an initialized VSL socket.
 *
 * Note that this function creates a new vfer socket when accepting
 * the connection. Therefore, the caller has to make sure that
 * vfer_close() is called both for the listening vfer_fd (that was
 * passed to vsl_socket()) and the new vfer_fd (which is available
 * with vsl_vferfd() after this function returns). The listening vfer
 * fd can be closed as soon as vsl_accept_ssh() returns, the accepted
 * vfer fd must not be closed before vsl_close() is called.
 *
 * @param[out] socket VSL socket to use
 *
 * \return
 *  - the vfer fd of the created connection on success (>= 0)
 *  - #VSL_BADPROT in case of a protocol error
 *  - #VSL_ERRNO if a system call failed (errno is set accordingly)
 *  - #VFER_TIMEOUT a timeout occured
 **/
int vsl_accept_ssh (vsl_sock* socket) {
    int port;
    unsigned char secret[VSL_KEYLEN];

    LOG("attempting ssh startup as server...");

    TRY(ssh_read_secret(secret));
    TRY(ssh_vfer_listen(socket, &port));
    TRY(ssh_send_port(port));
    TRY(ssh_vfer_accept(socket));
    TRY(ssh_daemonize());

    init_keys_s(socket, secret);

    socket->status = C_ESTABLISHED;
    socket->auth = VSL_AUTH_SSH;

    LOG("ssh startup successful.");
    return 0;
}


/**
 * Initializes a VSL socket using an existing and connected vfer fd.
 * Before the socket can be used to transfer data, an authentication
 * function has to be called to perform the handshake.
 *
 * @param[in] fd the VFER fd.
 * @param[out] socket VSL socket
 * \return
 *  - 0 on success
 *  - #VFER_BADSOCK if \a fd is invalid
 */
int vsl_socket(vsl_sock* socket, const vfer_fd fd) {

    LOG("initializing socket...");

    socket->vfd = fd;
    socket->status = C_INITIALIZED;
    socket->sent_nr = 1;
    socket->recv_nr = 0;
    socket->test_mode = false;

    // just to make sure the vfd is valid
    int nonblocking = 0;
    int ret = sizeof(int);
    int tmp;
    if ( (tmp=vfer_getsockopt(socket->vfd, VFERSO_NONBLOCK, &nonblocking, &ret))
         == VFER_BADSOCK )
        LOGFAIL(tmp, "received invalid vfer socket!");
    else if ( tmp != 0 )
        LOGFATAL("vfer_getsockopt failed: %s!", vfer_errortext(tmp));


    LOG("socket initialized.");
    return 0;
}


/**
 * Destructs the given VSL socket.
 *
 * This function does nothing at the moment, but it should be called
 * as soon as the VSL connection is no longer used in case clean is
 * neccessary in later versions.
 *
 * This function will not close the underlying vfer_fd, it has to be
 * closed manually or can be used for normal vfer communication.
 *
 * @param[in] sock the socket
 */
void vsl_close (vsl_sock* sock) {
    LOG("closed vsl socket.");
}



/**
 * Get the underlying VFER fd of a VSL socket.
 *
 * @param[in] socket
 * @return the vfer fd
 */
vfer_fd vsl_vferfd(vsl_sock* socket) {
    return socket->vfd;
}

/**
 * Get the maximum size that can be transmitted in a \c vsl_send() or
 * \c vsl_recv() call.
 *
 * @param[in] socket the VSL socket
 * @return
 *  - the maximum frame size in bytes > 0 on success
 */
ssize_t vsl_max_frame_size(vsl_sock* socket) {
    return ( (vfer_max_frame_size(socket->vfd) > 1024*1024*1024 ?
              1024*1024*1024 :
              vfer_max_frame_size(socket->vfd)) - PHEAD_SIZE);
}


/**
 * Sends \a len bytes from \a buf over a socket.
 *
 * Equivalent to vfer_send(), but uses the vfer security layer to
 * protect the message against modification and eavesdropping.
 *
 * Due to the used MAC algorithm, the maximum number of messages that
 * can be transferred with one socket is \c sizeof(uint64_t).
 * Afterwards, #VSL_MAXMSG is returned.
 *
 * The maximum number of bytes that can be transmitted in a single
 * message can be determined with \c vsl_max_frame_size().
 *
 * If the vfer socket is of type \c SOCK_STREAM, nonblocking IO is
 * not supported and #VFER_INVAL returned.
 *
 * @param[in] sock VSL socket to use
 * @param[in] buf buffer to send
 * @param[in] len length of the data to send
 * @return
 *  - len on success
 *  - #VSL_TOOBIG message is bigger than vsl_max_frame_size().
 *  - #VFER_UNCONN sock is not connected
 *  - #VFER_INVAL invalid buf or len
 *  - #VFER_WOULDBLOCK socket is non-blocking and requested operation
 *    would block
 *  - #VSL_MAXMSG maximum message count reached
 *  - #VSL_ERRNO if a system call failed (errno is set accordingly)
 */
ssize_t vsl_send(vsl_sock* sock, const void * buf, size_t len) {
    LOG("attempting to send %zd bytes...", len);

    // Check for nonblocking IO
    if ( !is_blocking(sock) &&
         vfer_is_stream(sock->vfd) )
        LOGFAIL(VFER_INVAL, "nonblocking IO not supported for streamed connections!");



    if ( sock->sent_nr == 0 )
         LOGFAIL(VSL_MAXMSG, "maximum message count reached!");

    // set actual packet length
    if ( len > vsl_max_frame_size(sock) )
        LOGFAIL(VSL_TOOBIG, "Message too large.");

    /* create header */
    phead_t msg;
    msg.type = P_DATA;
    FILL_MAGIC(msg.magic);
    msg.datalen = htonl(len);
    msg.msgnr = htonll(sock->sent_nr++);

    /* Serialize */
    size_t buflen;
    unsigned char *sbuf;
    if ( (sbuf = write_objs(&buflen,
                            (void*) &msg.msgnr, sizeof(msg.msgnr),
                            (void*) &msg.hash, sizeof(msg.hash),
                            (void*) &msg.magic, sizeof(msg.magic),
                            (void*) &msg.type, sizeof(msg.type),
                            (void*) &msg.datalen, sizeof(msg.datalen),
                            buf, len, NULL)) == NULL)
         LOGFAIL(VSL_ERRNO, "malloc failed: %s", strerror(err));

    /* Encrypt */
    LOG("encrypting...");
    unsigned char ctr[16];
    memset((void*) ctr, 0, 16);
    memcpy((void*) ctr, (void*) sbuf, sizeof(msg.msgnr));
    aes_ctr(ctr, sock->enc_send_key, sbuf + PHEAD_SIZE, len);

    /* Authenticate (including msgnr) */
    LOG("Calculating MAC...");
    unsigned char hash[sizeof(msg.hash)];
    memset((void*) ctr, 0, 16);
    memcpy((void*) ctr, (void*) sbuf, sizeof(msg.msgnr));
    memset(sbuf + sizeof(msg.msgnr), 0, sizeof(msg.hash));
    poly1305aes_authenticate(hash, sock->auth_send_key, ctr,
                             sbuf, buflen);
    memcpy(sbuf + sizeof(msg.msgnr), hash, sizeof(msg.hash));


    /* Send  */
    LOG("Sending total message of size %zd bytes.", buflen);
    int ret;
    if ( (ret=transfer(sock, (transmit_fn) vfer_send, sbuf, buflen, buflen)) < 0) {
        free(sbuf);
        return ret;
    }
    free(sbuf);

    LOG("message sent.");
    return len;
}


/**
 * Sends \a len bytes from file \a fd starting at \a offset
 * buf over socket \a sock.
 *
 * Equivalent to vfer_sendfile(), but uses the vfer security layer to
 * protect the message against modification and eavesdropping.
 *
 * Due to the used MAC algorithm, the maximum number of messages that
 * can be transferred with one socket is \c sizeof(uint64_t). So it
 * may happen that large files cannot be transferred completely. In
 * this case #VSL_MAXMSG is returned after the maximum number of bytes
 * has been transferred.
 *
 * @param[in] sock the VFS socket to use
 * @param[out] fd file descriptor for reading
 * @param[in] offset offset to start reading
 * \param[in] len number of bytes to write
 * @return
 *      - Returns the actual number of bytes sent on success (a positive number)
 *      - errors like vsl_send()
 */
ssize_t vsl_sendfile(vsl_sock* sock, int fd, off_t offset, size_t len) {
    return vfer__sendfile( (vfer_trnsm_fn) vsl_send, (vfer_trnsm_arg) sock,
                           vsl_max_frame_size(sock),fd, offset, len );
}


/**
 * Reads \a len bytes from \a buf on a connected socket.
 *
 * Equivalent to vfer_recv(), but uses the vfer security layer to
 * protect the message against modification and eavesdropping.
 *
 * Due to the used MAC algorithm, the maximum number of messages that
 * can be transferred with one socket is \c sizeof(uint64_t).
 *
 * The maximum number of bytes that can be transmitted in one message
 * can be determined with  vsl_max_frame_size().
 *
 * Note that if the incoming message is larger than \a len, the
 * message cannot be verified and #VSL_BADPROT will be returned.
 *
 * If the vfer socket is of type \c SOCK_STREAM, nonblocking IO is
 * not supported and #VFER_INVAL returned.
 *
 * @param[in] sock the socket to use
 * @param[out] buf a pointer to a buffer into which to receive
 * @param[in] len maximum length of data to receive into buf
 * @return
 *  - Returns the actual number of bytes sent on success (a positive number)
 *  - #VFER_UNCONN sock is not connected
 *  - #VFER_INVAL invalid buf or len.
 *  - #VFER_WOULDBLOCK socket is non-blocking and requested operation
 *    would block
 *  - #VSL_BADPROT in case of a protocol error, may also be caused by
 *    a forged message
 *  - #VSL_TOOBIG message is larger than \a len.
 *  - #VSL_ERRNO if a system call failed (errno is set accordingly)
 */
ssize_t vsl_recv(vsl_sock* sock, void *buf, size_t len) {
    unsigned char* rbuf;
    size_t to_read;

    // Check for nonblocking IO
    if ( !is_blocking(sock) &&
         vfer_is_stream(sock->vfd) )
          LOGFAIL(VFER_INVAL, "nonblocking IO not supported for streamed connections!");


    // Start reading a new message
    if ( (rbuf = malloc(vfer_max_frame_size(sock->vfd))) == NULL)
        LOGFAIL(VSL_ERRNO, "malloc failed: %s", strerror(err));

    // Determine bytes to read
    if ( vfer_is_stream(sock->vfd) ) {
        LOG("retrieving header...");
        to_read = PHEAD_SIZE;
    }
    else {
        LOG("trying to retrieve %zd + %zd bytes (body+header)...",
            len, PHEAD_SIZE);
        to_read = len + PHEAD_SIZE;
    }

    // Read header
    ssize_t rcnt;
    if ( (rcnt=transfer(sock, (transmit_fn) vfer_recv, rbuf, PHEAD_SIZE, to_read)) < 0) {
        free(rbuf);
        return rcnt;
    }

    /* Deserialize */
    phead_t head;
    unsigned char* body;
    if ( (body = read_objs(rbuf, rcnt,
                     (void*) &head.msgnr, sizeof(head.msgnr),
                     (void*) &head.hash, sizeof(head.hash),
                     (void*) &head.magic, sizeof(head.magic),
                     (void*) &head.type, sizeof(head.type),
                     (void*) &head.datalen, sizeof(head.datalen),
                           NULL)) == NULL ) {
        // We should have caugth this error above
        free(rbuf);
        LOGFATAL("message too short!");
    }
    head.datalen = ntohl(head.datalen);
    LOG("actual body length %zd bytes.", head.datalen);

    // Stream: Retrieve body
    if ( vfer_is_stream(sock->vfd) ) {
        LOG("retrieving body...");
        if ( (rcnt = transfer(sock, (transmit_fn) vfer_recv, body, head.datalen, head.datalen)) < 0 ) {
            free(rbuf);
            return rcnt;
        }
    }

    // Datagram: Check packet completeness
    else if ( head.datalen != rcnt - PHEAD_SIZE ) {
        free(rbuf);
        LOGFAIL(VSL_BADPROT, "invalid message received (too short)!");
    }

    // Check if buf is large enough
    if ( head.datalen > len ) {
        free(rbuf);
        LOGFAIL(VSL_TOOBIG, "message does not fit into given buf!");
    }

    // Check if message size conforms to the protocol
    if ( head.datalen > vsl_max_frame_size(sock) ) {
        free(rbuf);
        LOGFAIL(VSL_BADPROT, "invalid message received (too long)!");
    }

    /* Check magic */
    if ( !CHECK_MAGIC(head.magic) ) {
        free(rbuf);
        LOGFAIL(VSL_BADPROT, "invalid message received (wrong magic)!");
    }

    /* Check type */
    if ( head.type != P_DATA ) {
        free(rbuf);
        LOGFAIL(VSL_BADPROT, "invalid message received (wrong type)!");
    }

    /* Check message nr  */
    if ( ntohll(head.msgnr) <= sock->recv_nr ) {
        free(rbuf);
        LOGFAIL(VSL_BADPROT, "invalid message received (old msgnr)!");
    }
    sock->recv_nr = ntohll(head.msgnr);

    /* Check MAC */
    LOG("Checking MAC...");
    unsigned char ctr[16];
    memset((void*) ctr, 0, 16);
    memcpy((void*) ctr, (void*) rbuf, sizeof(head.msgnr));
    memset(rbuf + sizeof(head.msgnr), 0, sizeof(head.hash));
    if ( poly1305aes_verify(head.hash, sock->auth_recv_key, ctr,
                            rbuf, head.datalen + PHEAD_SIZE) == 0 ) {
        free(rbuf);
        LOGFAIL(VSL_BADPROT, "invalid message received (wrong MAC)!");
    }

    /* Decrypt */
    LOG("Decrypting body...");
    memset((void*) ctr, 0, 16);
    memcpy((void*) ctr, (void*) rbuf, sizeof(head.msgnr));
    aes_ctr(ctr, sock->enc_recv_key, body, head.datalen);

    /* Copy message */
    memcpy(buf, body, head.datalen);
    free(rbuf);

    LOG("message received.");
    return head.datalen;
}

/**
 * Reads \a len bytes from \a buf on a connected socket and writes
 * them into the specified fd, starting at \a offset.
 *
 * Equivalent to vfer_recvfile(), but uses the vfer security layer to
 * protect the message against modification and eavesdropping.
 *
 * Due to the used MAC algorithm, the maximum number of messages that
 * can be transferred with one socket is \c sizeof(uint64_t).
 * Afterwards, #VSL_MAXMSG is returned.
 *
 * @param[in] sock the VFS socket to use
 * @param[out] fd file descriptor for writing
 * @param[in] offset offset to start writing
 * \param[in] len number of bytes to read
 * @return
 *      - number of bytes read on success
 *      - errors like vsl_recv()
 */
ssize_t vsl_recvfile(vsl_sock* sock, int fd, off_t offset, size_t len) {
    return vfer__recvfile( (vfer_trnsm_fn) vsl_recv, (vfer_trnsm_arg) sock,
                           vsl_max_frame_size(sock),fd, offset, len);
}


/**
 * Marks socket with the conditions to be tested by the select call.
 *
 * Called prior to using vsl_select(). Mark is a bitwise OR of
 *  - #VSL_READABLE if the socket has received a message
 *  - #VSL_WRITABLE if the socket can send a message
 *  - #VSL_AUTHABLE if the socket is ready for the next
 *    authenticiation step by one of the VLS authentication functions.
 *  - #VSL_EXCEPTION if an error occured for the socket
 *
 * @param[in] socket VSL socket to mark
 * @param[in] mark a mark to associate with the socket
 * @return
 *      - 0 on succes
 *      - #VFER_INVAL invalid mark value
 */
int vsl_selectmark(vsl_sock* socket, int mark) {
    LOG("entered.");
    if ((mark & ~(VSL_READABLE | VSL_WRITABLE | VSL_EXCEPTION | VSL_AUTHABLE)) != 0)
        LOGFAIL(VFER_INVAL, "invalid mark value!");

    socket->selectmark = mark;
    LOG("marked socket.");
    return 0;
}

/**
 * Returns the result of the vsl_select() call for socket.
 *
 * The return value will be a bitwise OR of the constants described in
 * vsl_selectmark(), depending on how \a sock was marked and the
 * result of the vsl_select() call.
 *
 * @param[in] sock the VSL socket
 * @return
 *      - bitwise OR of marks for socket \a sock on succes
 *      - #VFER_BADSOCK bad socket
 */
int vsl_selecttest(vsl_sock* sock) {
    return sock->selectres;
}

/**
 * Returns the number of VSL sockets in an array that satisfy the
 * marked conditions.
 *
 * \a timeout is the maximum time to wait before the call returns. If
 * \a timeout is \c NULL the call blocks indefinitely. A timeout value of
 * zero can be used to effect a poll operation.
 *
 * \warning Due to some implementation problems, this function might
 * return immediately with #VFER_INPROGRESS. That means that at the
 * moment no socket is ready, and that it is not possible to wait for
 * all specified sockets at the same time. In this case, the
 * application has to poll the sockets by repetedly calling this
 * function. However, this behaviour can only occur if \a len is
 * bigger than 1 and at least one socket is marked with #VSL_AUTHABLE.
 *
 * @param[in] len     length of socks array
 * @param[in] socks   an array of sockets to check for marked conditions
 * @param[in] stimeout a timeout value to control waiting time
 * @return
 *    - a positive number if some sockets satisfy the marked
 *      conditions (\b not the number of sockets).
 *    - #VFER_INVAL timeout value is invalid
 *    - #VFER_INPROGRESS blocking select not possible
 *    - #VSL_ERRNO if a system call failed (errno is set accordingly)
 */
int vsl_select(int len, vsl_sock** socks, struct timeval *stimeout) {
    LOG("entered.");

    vfer_fd vfer_fds[len];
    fd_set read_fds, write_fds, except_fds;
    int vfer_fd_count=0,
        file_fd_count=-1,
        pid_count=0;

    FD_ZERO(&read_fds);
    FD_ZERO(&write_fds);
    FD_ZERO(&except_fds);

    /* Loop over all sockets to determine what we're waiting for */
    for (int i=0; i < len; i++) {
        vsl_sock* sock = socks[i];
        sock->selectres = 0;

        /* Connected sockets can only be ready for reading,
         * writing or exceptions */
        if ( sock->status == C_ESTABLISHED ) {
            int mark = 0, ret;
            mark |= VSL_READABLE  & sock->selectmark ? VFER_READABLE  : 0;
            mark |= VSL_WRITABLE & sock->selectmark ? VFER_WRITABLE : 0;
            mark |= VSL_EXCEPTION & sock->selectmark ? VFER_EXCEPTION : 0;
            if ((ret=vfer_selectmark(sock->vfd, mark)) < 0)
                LOGFATAL("unknown vfer error: %s", vfer_errortext(ret));
            vfer_fds[vfer_fd_count++] = sock->vfd;
            LOG("adding socket to vfer select list.");
            continue;
        }

        /* If the socket is not yet connected, we have to
         * check the current state */
        if ( sock->selectmark & VSL_AUTHABLE ) {
            int ret;

            // Ready to start auth, no need for select
            if (sock->status == C_INITIALIZED) {
                sock->selectres = VSL_AUTHABLE;
                LOGFAIL(1, "socket in initialized state, returning now.");
            }

            switch (CONWAIT(sock->status)) {

            case C_WAIT_FD_R:
                // Waiting to read from fd
                FD_SET(sock->ssh_stdout_fd, &read_fds);
                FD_SET(sock->ssh_stdout_fd, &except_fds);
                LOG("adding socket to read fd set.");
                if ( sock->ssh_stdout_fd > file_fd_count )
                    file_fd_count = sock->ssh_stdout_fd;
                continue;

            case C_WAIT_FD_W:
                // Waiting to write to fd
                FD_SET(sock->ssh_stdin_fd, &write_fds);
                FD_SET(sock->ssh_stdin_fd, &except_fds);
                LOG("adding socket to write fd set.");
                if ( sock->ssh_stdin_fd > file_fd_count )
                    file_fd_count = sock->ssh_stdin_fd;
                continue;

            case C_WAIT_PID:
                // Waiting for process
                if ( sock->ssh_pid == 0 )
                    LOGFAIL(1, "already waited for pid, returning now.");
                pid_count++;
                LOG("adding socket to pid wait set.");
                continue;

            case C_WAIT_VFD_R:
                // Waiting to read vfer packet
                if ((ret=vfer_selectmark(sock->vfd, VFER_READABLE | VFER_EXCEPTION)) < 0)
                    LOGFATAL("unknown vfer error: %s", vfer_errortext(ret));
                vfer_fds[vfer_fd_count++] = sock->vfd;
                LOG("adding socket to vfer select list.");
                continue;


            case C_WAIT_VFD_W:
                // Waiting to send vfer packet
                if ((ret=vfer_selectmark(sock->vfd, VFER_WRITABLE | VFER_EXCEPTION)) < 0)
                    LOGFATAL("unknown vfer error: %s", vfer_errortext(ret));
                vfer_fds[vfer_fd_count++] = sock->vfd;
                LOG("adding socket to vfer select list.");
                continue;

            }
        }
    }
    file_fd_count++;


    /* Test if we have to poll */
    struct timeval *timeout;
    if ( (vfer_fd_count != 0 &&
          file_fd_count != 0) ||
         (vfer_fd_count != 0 &&
          pid_count != 0) ||
         (file_fd_count != 0 &&
          pid_count != 0) ) {
        LOG("polling neccessary.");
        timeout = &((struct timeval) { 0, 0 });
    }
    else {
        LOG("no polling neccessary.");
        timeout = stimeout;
    }


    /* Select VFER sockets */
    int ret, cnt=0;
    if (vfer_fd_count) {
        LOG("calling vfer_select...");
        if ( (cnt = vfer_select(vfer_fd_count, vfer_fds, timeout)) == -1 )
            LOGFAIL(VSL_ERRNO, "vfer_select failed: %s!", vfer_errortext(cnt));
        LOG("%d sockets ready so far..", cnt);

        /* Save marks */
        for(int i=0; i < len; i++) {
            vsl_sock* sock = socks[i];
            if ( sock->status == C_ESTABLISHED ) {
                if ( (ret = vfer_selecttest(sock->vfd)) < 0)
                    LOGFATAL("unknown vfer error: %s", vfer_errortext(ret));
                if ( ret & VFER_READABLE )
                    sock->selectres |= VSL_READABLE;
                if ( ret & VFER_WRITABLE )
                    sock->selectres |= VSL_WRITABLE;
                if ( ret & VFER_EXCEPTION )
                    sock->selectres |= VSL_EXCEPTION;
            }


            if ( sock->selectmark & VSL_AUTHABLE &&
                 ( CONWAIT(sock->status) == C_WAIT_VFD_W ||
                   CONWAIT(sock->status) == C_WAIT_VFD_R ) ) {
                if ( (ret = vfer_selecttest(sock->vfd)) < 0)
                    LOGFATAL("unknown vfer error: %s", vfer_errortext(ret));
                if (ret)
                    sock->selectres |= VSL_AUTHABLE;
            }
        }
    }


    /* Select FIFOs */
    if (file_fd_count) {
        LOG("calling select...");
        if ( (cnt += select(file_fd_count, &read_fds, &write_fds, &except_fds,
                            timeout)) == -1)
            LOGFAIL(VSL_ERRNO, "select call failed: %s!", strerror(errno));
        LOG("%d sockets ready so far..", cnt);

        /* Save marks */
        for(int i=0; i < len; i++) {
            vsl_sock* sock = socks[i];
            if ( sock->selectmark & VSL_AUTHABLE &&
                 CONWAIT(sock->status) == C_WAIT_FD_W )
                if ( FD_ISSET(sock->ssh_stdin_fd, &write_fds) ||
                     FD_ISSET(sock->ssh_stdin_fd, &except_fds) )
                    sock->selectres |= VSL_AUTHABLE;

            if ( sock->selectmark & VSL_AUTHABLE &&
                 CONWAIT(sock->status) == C_WAIT_FD_R )
                if ( FD_ISSET(sock->ssh_stdout_fd, &read_fds) ||
                     FD_ISSET(sock->ssh_stdout_fd, &except_fds) )
                    sock->selectres |= VSL_AUTHABLE;
        }

    }


    /* Check PIDs */
    if (pid_count) {
        LOG("Checking PIDs...");
        for(int i=0; i < len; i++) {
            vsl_sock* sock = socks[i];
            if ( (sock->selectmark & VSL_AUTHABLE) &&
                 CONWAIT(sock->status) == C_WAIT_PID) {
                LOG("trying to wait for pid %d..", sock->ssh_pid);
                // We ignore errors here, they are handled in
                // ssh_close()
                if( waitpid(sock->ssh_pid, &(sock->ssh_pid_stat), WNOHANG) > 0) {
                    LOG("successful.");
                    sock->ssh_pid = 0;
                    cnt++;
                    sock->selectres |= VSL_AUTHABLE;
                }
                else
                    LOG("still running.");
            }
        }
        LOG("%d sockets ready so far..", cnt);
    }


    LOG("returning.");
    if (timeout == stimeout || cnt)
        return cnt;
    else
        return VFER_INPROGRESS;
}



/**
 * Returns a text description of the error code \a err. If the
 * errorcode is not specific to the security api, vfer_errortext() is
 * called automatically instead.
 *
 * @param[in] err error code
 * @return
 *      a pointer to a text description of \a err
 */
char* vsl_errortext(int err) {
    switch (err) {
    case VSL_BADPROT:
        return "protocol error or forged message";
    case VSL_MAXMSG:
        return "maximum message count exceeded";
    case VSL_ERRNO:
        return strerror(errno);
    case VSL_SSHF:
        return "ssh subprocess failed";
    case VSL_TEMP:
        return "server signaled temporary error";
    case VSL_PERM:
        return "server signaled permanent error";
    case VSL_TOOBIG:
        return "message too big";
    default:
        return vfer_errortext(err);
    }
}


/**
 * Activates debugging messages
 *
 * \param[in] fp stream to send debugging messages to, \c NULL deactivates
 * debugging.
 * \param[in] vfer If not \c NULL, configures vfer to send its
 * debugging output (level given by the contents of \a vfer) to the
 * same file.
 * */
FILE* debug = NULL;
void vsl_debug (FILE* fp, const char* vfer) {
    // Handle disabling
    if (fp == NULL) {
        debug = NULL;
        vfer_debug(NULL, NULL, "");
        return;
    }

    /* dup the fd so it isn't lost when we fork */
    int fd;
    if ( (fd = dup(fileno(fp))) == -1 ) {
        fprintf(stderr, "Cannot dup fd: %s!\n", strerror(errno));
        abort();
    }
    if ( (debug = fdopen(fd, "a")) == NULL ) {
        fprintf(stderr, "Cannot fdopen fd %d: %s!\n", fd, strerror(errno));
        abort();
    }
    if (vfer == NULL)
        vfer = "";
    vfer_debug(debug, debug, vfer);
}

/**
 * Activates some behaviour that is only useful for testing
 * VSL internals:
 *
 *  - vsl_connect_ssh() returns after each step in non blocking mode
 *
 * \param[in] sock socket to mark
 **/
void vsl_test_mode(vsl_sock* sock) {
    LOG("Activating test mode.");
    sock->test_mode = true;
}

/**
 * Like vsl_select(), but automatically polls every \a usec
 * microseconds when vsl_select() would return #VFER_INPROGRESS.
 *
 * @param[in] len     length of socks array
 * @param[in] socks   an array of sockets to check for marked conditions
 * @param[in] stimeout a timeout value to control waiting time
 * @param[in] usec polling intervall in microseconds
 * @return
 *    - number of sockets in array socks that satisfy the marked conditions
 *    - #VFER_INVAL timeout value is invalid
 *    - #VSL_ERRNO if a system call failed (errno is set accordingly)
 */
int vsl_selectpoll(int len, vsl_sock** socks, struct timeval *stimeout, int usec) {
    int ret;
    struct timeval start;
    struct timeval now;
    GET_TIME_OF_DAY(&start);
    do {
        ret = vsl_select(len, socks, stimeout);
        if ( ret == VFER_INPROGRESS ) {
            GET_TIME_OF_DAY(&now);
            if ( TIMEVAL_MS(*stimeout) > TIMEVAL_MS_DIFF(start, now) )
                return 0;
            usleep(usec);
            continue;
        }
        else
            return ret;
    } while(1);
}





/*
 * Local Variables:
 * compile-command: "cd ..; make tests file_xfer"
 * compilation-search-path: ("..")
 * End:
 */

---