src/vfer.h

Go to the documentation of this file.

/*
 * Copyright 2005, 2006, Internet2
 * Legal conditions are in file LICENSE
 * (MD5 = c434f2e53b8089d8b4d0172c7ce07360).
 */
/**
 *
 * @file   vfer.h
 * @author Ivan Beschastnikh
 * @brief  Header file that is used by multiple sources within the protocol stack.
 *
 * This header file includes many useful macros & functions for error
 * processing, memory allocation and more.
 *
 * -     12/28/05        ivan            tsci2 integration ; rel func prototype def and context structure ; numerous changes to ccontrol struct and others
 * -     09/01/05        ivan            added packet debugging macro and packet alloc and release macros
 * -     07/28/05        ivan            modified to support c_control socket links + added some defines
 * -     05/21/05        ivan            created
 */


#ifndef VFER_H
#define VFER_H

/*
 * includes
 */

#include <stdlib.h>
#include <stdio.h>

#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <string.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <pthread.h>    /* thread support => compile with -lpthread */
#include <signal.h>
#include <sys/select.h> /* select */
#include <sys/mman.h>   /* mmap*/
#include <errno.h>      /* error functionality */
#include <sys/select.h>
#include <math.h>
#include <fcntl.h>
#include <sys/time.h>
#include <time.h>

#ifdef TSCI2
// #include <tsci2.h>
#include "/home/ivan/include/tsci2.h"
#endif

#include "vfer_btree_delay.h"
#include "vfer_accept_queue.h"
#include "vfer_datagram.h"
#include "vfer_packet.h"

/*
 * constants
 */

/* note that version MUST be less than 16 - see packetizing of the
 * vfer header in the spec rev. 1.7 and greater */
#define VFER_VERSION            1

/* debug\error defs */

/* these can be reset to point to open files, but they are not used if
 * DEBUG_ERR and/or DEBUG_OUT are commented out */
FILE* debug_out;
FILE* error_out;

/* determines whether we output any err msgs to debug_err */
#define DEBUG_ERR
/* determines whether we output any informational msgs to debug_out */
#define DEBUG_OUT

/*
 * these vars determine the printing of messages for specific layers
 * of the protocol (they are set via vfer_debug() in api.c)
 */
char DEBUG_ACCEPTQ;
char DEBUG_API;
char DEBUG_CTL;
char DEBUG_CCTL;
char DEBUG_PACKET;
char DEBUG_PMTUD;

/**/
#ifdef DEBUG_ERR
        #define ERROR_PRINT(FILE, FUNC, ...) { \
                fprintf(error_out, "ERROR [%s :: %s] ", FILE, FUNC); \
                fprintf(error_out, __VA_ARGS__); \
                fprintf(error_out, "\n"); \
                fflush(error_out); }
#else
        #define ERROR_PRINT(COND, FILE, FUNC, ...)
#endif
/**/
#ifdef DEBUG_OUT
        #define DEBUG_PRINT(COND, FILE, FUNC, ...) \
                if (COND && debug_out != NULL) {fprintf(debug_out, "%-13s [ %-20s ] ", #COND, FUNC); \
                fprintf(debug_out, __VA_ARGS__); \
                fprintf(debug_out, "\n"); \
                fflush(debug_out); }
        #define DEBUG_PACK(COND, FILE, FUNC, PACKET, ...) \
                if (COND && debug_out != NULL) {fprintf(debug_out, "%-13s [ %-20s ] ", #COND, FUNC); \
                fprintf(debug_out, __VA_ARGS__); \
                fprintf(debug_out, "\n%-50s %s\n", " ", Packet_Str(PACKET)); \
                /* fprintf(debug_out, "%s\n", Packet_Str(PACKET)); */ \
                fflush(debug_out); }
        #define DEBUG_CC(COND, FILE, FUNC, SOCKET, ...) \
                if (COND && debug_out != NULL) { \
                    struct timeval tv; \
                    GET_TIME_OF_DAY(&tv); \
                    fprintf(debug_out, "%-13s [ %-20s ] ", #COND, FUNC); \
                    fprintf(debug_out, "rtt[%u] sendw[%u,%u] cwnd[%d] inflight[%u] recvw[%u,%u] rlost[%u] delay[dlta %d]\n", \
                            (RTT(SOCKET,tv) == 0 ? 1 : RTT(SOCKET,tv)), SOCKET->ccontrol.s_first_seq, SOCKET->ccontrol.s_last_seq, SOCKET->ccontrol.s_cwnd, \
                            SOCKET->ccontrol.s_inflight, SOCKET->ccontrol.r_first_seq, SOCKET->ccontrol.r_last_seq, SOCKET->ccontrol.r_packets_lost, \
                            SOCKET->ccontrol.delay_delta); \
                    fflush(debug_out);   \
                }
        #define DEBUG_PMTUD_PRINT(COND, FILE, FUNC, ...) \
                if (COND && debug_out != NULL) {fprintf(debug_out, "%-13s [ %-20s ] ", #COND, FUNC); \
                        fprintf(debug_out, __VA_ARGS__); \
                        fprintf(debug_out, "\n"); \
                        fflush(debug_out);}
#else
        #define DEBUG_PRINT(COND, FILE, FUNC, ...)
        #define DEBUG_PACK(COND, FILE, FUNC, PACKET, ...)
        #define DEBUG_CC(COND, FILE, FUNC, SOCKET, ...)
        #define DEBUG_PMTUD_PRINT(COND, FILE, FUNC, SOCKET, ...)
#endif


#define PRINT_MEM printf("MEMORY USED [%d bytes]\n", (int)ReqMem);

/* useful utility macros */

/**/
#ifdef TSCI2
        #define GET_TIME_OF_DAY(TV_PTR) tsci2_gettimeofday(TV_PTR, NULL);
#else
        #define GET_TIME_OF_DAY(TV_PTR) gettimeofday(TV_PTR, NULL);
#endif

/* evaluates to the value of TV in milliseconds where TV is of type struct timeval */
#define TIMEVAL_MS(TV) ((unsigned int)(((TV).tv_sec * 1000) + ((TV).tv_usec / 1000)))
#define TIMEVAL_US(TV) ((unsigned int)(((TV).tv_sec * 1000000) + ((TV).tv_usec)))

/* evaluates to (TV2 - TV1) in milliseconds where TV1 and TV2 are of type struct timeval */
#define TIMEVAL_MS_DIFF(TV1, TV2) (TIMEVAL_MS(TV2) - TIMEVAL_MS(TV1))

/* evaluates to (TV2 - TV1) in microseconds where TV1 and TV2 are of type struct timeval */
#define TIMEVAL_US_DIFF(TV1, TV2) (TIMEVAL_US(TV2) - TIMEVAL_US(TV1))

/* computes round trip time in us, TV is the current time stamp which is used by the btree to prune old delay values */
#define RTT(SOCK_PTR, TV) abs(SOCK_PTR->ccontrol.delay - Tree_Min(&(SOCK_PTR->ccontrol.rev_path_curr_delayhist), TV))

#define ALLOC(DST_ADDR,TYPE_POINTER,TYPE,NUM) ({ \
        DST_ADDR=NULL; \
        if ((DST_ADDR = (TYPE_POINTER) calloc(NUM, sizeof( TYPE )))) \
             ReqMem += (sizeof(TYPE) * NUM); \
        /* printf("ALLOC[%d]\n", ReqMem); */ \
        DST_ADDR; })

#define RELEASE(POINTER_TO_BLOCK, SIZE) { \
        free(POINTER_TO_BLOCK); \
        ReqMem -= SIZE; \
        /* printf("FREE[%d]\n", ReqMem); */ }\

#define RELEASE_IOV(IOV, LEN) { \
        int release_iov_itter; \
        for(release_iov_itter = 0; release_iov_itter < LEN; release_iov_itter++) { \
                /* printf(" : iov[%d] len[%d] : ", release_iov_itter, IOV[release_iov_itter].iov_len); */ \
                RELEASE(IOV[release_iov_itter].iov_base, IOV[release_iov_itter].iov_len); } \
        RELEASE(IOV, sizeof(struct iovec));}

#define RELEASE_PACK(PACK) { \
        /* printf("releasing pack with iovlen[%d] : ", PACK->iovlen); */ \
        RELEASE_IOV(PACK->iov, PACK->iovlen); \
        RELEASE(PACK, sizeof(struct packet));}

#define UNLOCK_AND_RETURN_ERROR(mutex, err) { \
        pthread_mutex_unlock(&mutex); \
        return err; }

#define SET_ERROR_RETURN(sock, error_type, ret) { \
        if (sock != NULL) sock->err = error_type; \
        return ret; }

#define SET_ERROR_UNLOCK_RETURN(sock, error_type, mutex_ptr, ret) { \
        if (sock != NULL) sock->err = error_type; \
        pthread_mutex_lock(mutex_ptr); \
        return ret; }

/* INIT_VAL is in microseconds */
#define TREE_INIT(TREE_PTR, INIT_VAL, TV) { \
                if (INIT_VAL >= 1000000) { \
                        TV.tv_sec = (int)(INIT_VAL / 1000000.0); \
                } else { \
                        TV.tv_sec = 0; \
                } \
                TV.tv_usec = (INIT_VAL - TV.tv_sec * 1000000); \
                Tree_Init(TREE_PTR, TV); }

#define CLR_SOCKET_STATS(stats_ptr) { bzero(stats_ptr, sizeof(struct vfer_stats)); }

/* socket util macros for on\off socket options only */
#define SOCK_OPT_ISSET(socket, opt) (socket->opts & opt)


/* protocol control constants */

/* server */
#define MAX_RESPONSE_TRIES              5
#define HANDSHAKE_ACK_WAIT_INTERVAL     1000000                         /* microsecs to wait for an ack after sending a response (before resending a response) */

/* client */
#define MAX_REQUEST_TRIES               5
#define REQUEST_RETRY_INTERVAL          1000000                         /* microseconds */

/* default transfer size constants (for now) */
#define INIT_MTU                        1024  /* initial MTU for a VFER connection */
#define MAX_FRAME_SIZE                  65535 /* this is the max UDP datagram size as well */
#define URGENCY_THRESHOLD               1     /* in frames ; accounts for reordering */

/* default Path MTU constants */
#define COMMON_MTU_LIST { 1024, 1430, 1978, 4282, 8096, 17844, 31030, 65465 } /* List of common MTUS, last value should be no larger than MAX_FRAME_SIZE */
#define COMMON_MTU_MAX_INDEX 7                     /* Number of common MTUS - 1 */
#define NO_PROBE_SENT 0
#define PROBE_SENT 1
#define MTU_FOUND 2
#define NO_PMTUD 3
#define PMTUD_TIME_INTERVAL 600 /* Time in seconds between probes after MTU found. */

/* define ICMP constants */
#define DEST_UNREACHABLE 3
#define FRAG_NEEDED 4

/* flow control buffer[s] size constants */
#define DEFAULT_RECV_BUF_MAXPACKETS             3000 /* number of packets ; should be same as SEND_BUF_MAXPACKETS */
#define DEFAULT_SEND_BUF_MAXPACKETS             3000 /* number of packets */

/* in bytes */
#define CALC_UDP_RECV_BUF_SIZE(MAXPACKETS) ((DATA_LENGTH + INIT_MTU)*MAXPACKETS)
#define CALC_UDP_SEND_BUF_SIZE(MAXPACKETS) ((DATA_LENGTH + INIT_MTU)*MAXPACKETS)

#define MMAP_CHUNK_SIZE                 50000000 /* 50mb : mmap chunk size for vfer_sendfile to use or rather the closeset page_size multiple to this value */
#define SEQ_WINDOW                      5 /* not used right now */
#define FRAME_WINDOW                    5 /* not used right now: meant to discard frames that are too far away */

/* control thread sleeping\activity cycle or rate of waking up */
#define CONTROLT_SLEEPTIME              1000 /* in us ; main control loop sleep time ; if this changes, RTT comparisons in ControlT need to change as well */

/* flow control restriction: max number of bytes that can be enqueued in the send queue */
#define MAX_SEND_WINDOW                 1024*8000 /* 8mb */

/* default timeouts on vfer_* calls in microseconds */
#define DEFAULT_CONNECT_TIMEOUT         60000000 /* 60s vfer_connect() */

/* CCONTROL VARIABLES */
#define UNACKED_THRESH                          4 /* MTU factor of unacked data to receive before sending an ack */
#define MAX_CWND_INCREASE_PACKETS_PER_RTT       8
#define CCONTROL_TARGET                         8000 /* in microseconds, 8(ms) target */

/*
 * data structures
 */

typedef struct sockaddr SA;

/**
 * @brief congestion control structure maintained for each socket
 */
typedef struct ccontrol {
        /* sending vars */
        uint32_t s_first_seq;           /* oldest unacknowledged (implicitly or explicitly) sequence number that was sent */
        uint32_t s_last_seq;            /* last sequence number sent */
        int32_t s_cwnd;                 /* current congestion control sending window size (in bytes) */
        uint32_t s_cwnd_sent;           /* number of bytes sent in the s_cwnd in the 'current' RTT */
        int32_t s_maxw;         /* maximum (!)congestion sending window size (in bytes) */
        uint32_t s_inflight;            /* number of packets in flight (s_last_seq - s_first_seq) */
        struct timeval s_last_ack;      /* timestamp of send of last ack */
        struct timeval s_rtt_start;     /* timestamp of the start of the current rtt : when this becomes older than RTT, we reset s_cwnd_sent to 0 and set s_rtt_start to curren time */
        struct timeval s_last_unacked_data;     /* timestamp of last sent data packet and not receiving any ack afterwards ; set to 0 if did not send any data since receipt of last ack */
        frame_link* s_last_data_frame;  /* last data packet frame pointer */

        uint32_t s_last_data_offset;    /* the offset of the last data sent (we don't save the length because the MTU might vary) */

        struct timeval s_last_vec;      /* timestamp of last sending of a vec ack packet */
        uint16_t s_need_ccack;          /* binary valued var to tell us if need to send an ack in this RTT period */

        /* receiving vars */
        uint16_t r_packets_lost;        /* number of packets lost since last sent cc ack */
        uint32_t r_first_seq;           /* first unacked seq received */
        uint32_t r_last_seq;            /* last seq received */
        uint32_t r_bytes_total;         /* total number of data bytes received for this connection */
        struct timeval r_last_ccack;    /* timestamp of receipt of last ccack packet */
        struct timeval r_last_data;     /* timestamp of receipt of last data packet */
        uint32_t r_unacked_bytes;       /* number of bytes that haven't been acked yet (since sending previous ack) */

        uint16_t last_packets_lost;     /* the last packets_lost received for this connection */
        uint32_t last_bytes_total;      /* the last total data bytes advertised as received by the other end host */

        uint32_t mtu;                   /* MTU for the sending path */
        uint32_t rtt;                   /* us current rtt estimate */
        int32_t delay_delta;            /* this is the value received on last ccack packet and is (base_delay - curr_delay) for the sending path (computed and maintained at the sender) */
        int32_t delay;                  /* this is the value received on last ccack packet and is the current delay for the sending path */
        tree_t rev_path_base_delayhist; /* delay history structure that maintains min delay over a 'long' time period (on order of seconds) for the reverse path */
        tree_t rev_path_curr_delayhist; /* delay history structure that maintains min delay over a shorter time period (on order of ms) for the reverse path */
        int syncing;                    /* set when the next data pack to send requires a sync option to be set */
        struct timeval r_last_loss;     /* timestamp of last loss event received ; this val reset (ie. tv_sec=0) once the timestamp is older than RTT */
} ccontrol_t;

/**
 * @brief VFER stats structure maintained for each socket
 */
typedef struct vfer_stats {
        uint32_t packets_recvd;         /* total packets received */
        uint32_t packets_sent;          /* total packets sent */
        uint32_t bytes_recvd;           /* total bytes received (all packets) */
        uint32_t bytes_sent;            /* total bytes sent (all packets) */
        uint32_t data_bytes_recvd;      /* total bytes received (data payload only) */
        uint32_t data_bytes_sent;       /* total bytes sent (data payload only) */
        struct timeval tv_start;        /* the start time of the connection : once the connection has been established (after connecting handshake) */
        struct timeval tv_end;          /* the end time of the connection : once the connection has been closed (after closing handshake) */
        double avg_rtt;                 /* is us ; avg round trip time recorded for the conn*/
        double max_rtt;                 /* is us ; max round trip time recorded for the conn */
        double min_rtt;                 /* is us ; min round trip time recorded for the conn */
        uint32_t rtt_measurements;      /* number of rtt measurements taken */
        uint32_t data_bytes_sender_sent;/* data payload only : bytes that were sent with the vfer_send call (ie. sent by the sender, not the protocol) */
} vfer_stats;

/**
 * @brief context structure used by the reliability functino to evaluate reliability for a frame
 */
typedef struct context_t {      /* reliability function's argument/context */
        frame f;
        vfer_stats* stats;
        frame_stats* fstats;
} context_t;

typedef uint32_t bool_t;                /* reliability function return value: 0,1 */
typedef bool_t(*relfun_t)(context_t*);  /* reliability function type: takes context pointer type and return bool type */

/**
 * @brief vfer_error type
 */
enum vfer_error {
        VFER_BADTYPE=-40,
        VFER_IMPL,
        VFER_BADSOCK,
        VFER_TIMEOUT,
        VFER_REFUSED,
        VFER_NOCONN,
        VFER_NOBIND,
        VFER_BADADDR,
        VFER_NOLISTEN,
        VFER_NOACCEPT,
        VFER_NOTSTREAM,
        VFER_WOULDBLOCK,
        VFER_BADOPT,
        VFER_BADFD,
        VFER_INVAL,
        VFER_UNCONN,
        VFER_INPROGRESS
} vfer_error;

typedef enum sock_mark {
        VFER_READABLE=1,
        VFER_WRITABLE=2,
        VFER_EXCEPTION=4
} sock_mark_t;

typedef enum sock_state {
        CONN_ACQUIRED,          /* allocated/reserved socket with the vfer_socket() call */
        CONN_BOUND,             /* bound socket (to a port number) */
        CONN_LISTENING,         /* socket listening for incoming connections */
        CONN_ACCEPTING,         /* socket is accepting a connection (has a reference to a socket in CONN_LISTENING state) */
        CONN_CONNECTING,        /* handshake state */
        CONN_CONNECTED, /* active connection duplex open on both sides local/remote */
        CONN_DISCONNECTING,     /* means the connection will be closed once the send queue drains */
        CONN_DISCONNECTED,      /* socket does not send/recv & has no
                                 * datagrams allocated : error to
                                 * recv/send on this socket */
        CONN_ERR                /* socket with an error- connection
                                 * must be killed without any sending
                                 * or receiving (for now immediate,
                                 * later with timeout?) */
} sock_state;

typedef enum sock_opt {
        VFERSO_QTTL             = 1,
        VFERSO_SNDSIZE          = 3,
        VFERSO_RELFUN           = 5,

        VFERSO_CONNECTTIMEOUT   = 6,
        VFERSO_CLOSETIMEOUT     = 7,
        VFERSO_ACCEPTTIMEOUT    = 9,
        VFERSO_SNDTIMEOUT       = 10,
        VFERSO_RCVTIMEOUT       = 11,

        /* these have to be powers of 2 as they will be incoded as
         * bits into an int (vfer_sock.opts) */
        VFERSO_NOPMTUD          = 2,
        VFERSO_NONBLOCK         = 4,
        /* this option is client read-only */
        VFERSO_CLIENT_SOCK      = 8, /* indicates that this sock is a client socket that 'connected' rather than being 'accepted' */
} sock_opt;

/**
*@brief Structure holding values related to path MTU discovery
*/
typedef struct path_mtu {
        char probe_state;       /* 0 = no probe sent, 1 = probe_sent, 2 = MTU found, 3 = NO PMTUD */
        int common_mtus[COMMON_MTU_MAX_INDEX + 1]; /* List of common mtus to use as search space */
        int* search_high;       /* During search, maximum possible value for path mtu */
        int* search_low;        /* During search, minimum possible value for path mtu */
        int* probe_size;        /* Size of the current probe sent or that will be sent */
        uint32_t frame_num;     /* Number of frame containing path MTU probe */
        uint32_t data_offset;   /* Offset of probe packet in frame */
        time_t next_probe_time; /* Time at which the next probe will be conducted */
} path_mtu;

/**
 * @brief vfer socket structure holding everything associated with a single vfer connection
 */
typedef struct vfer_sock {
        int fd;                 /* file descriptor for the socket after we are connected or for a server that is listening*/
        int icmp_fd;            /* file descriptor to listen for ICMP messages used in PMTU discovery*/
        int type;               /* SOCK_STREAM or SOCK_DGRAM */
        sock_state state;       /* state of the socket */
        int opts;               /* options turned ON\OFF for this socket */
        int err;                /* error for this socket */
        struct sockaddr_in addr;/* remote address this socket is connected */
        socklen_t addr_len;     /* length of addr field */
        vfer_stats stats;       /* stats for this socket */
        uint32_t l_seq;         /* last local seq number + 1 */
        uint16_t l_frame_num;   /* last local frame number */
        uint32_t r_seq;         /* last remote seq number + 1 */
        uint16_t r_frame_num;   /* last remote frame number */
        uint32_t r_opts;                /* remote opts */
        uint32_t r_recv_buf_size;       /* remote receive buffer size in packets */
        uint32_t r_send_buf_size;       /* remote send buffer size in packets */
        uint32_t r_max_frame_size;      /* remote max frame size in bytes */
        relfun_t relfun;                /* reliability function to use */

        int select_active;              /* 1(active): implies that ControlT should check select_mark and set result_mark & signal necessary condition var if a condition is satisfied, 0: ignore mark */
        sock_mark_t select_mark;        /* mark for vfer_select to test for */
        sock_mark_t select_result;      /* resultant mark to be read from after vfer_select() is done */
        pthread_mutex_t mutx_readable;
        pthread_cond_t cond_readable;
        pthread_mutex_t mutx_writeable;
        pthread_cond_t cond_writeable;

        pthread_mutex_t mutex;          /* access mutex */
        pthread_cond_t cond;            /* used for signaling closure/connection ready to be accepted/connection (ie. CONNECTED state) all set in ControlT main thread loop */
        uint32_t udp_send_buf_size;     /* flow control */
        uint32_t udp_recv_buf_size;     /* flow control */

        ccontrol_t ccontrol;            /* congestion control structure containing all the necessary information to form ccacks and control sending window */

        frame_list send_frames;         /* frames we are sending */
        frame_list recv_frames;         /* frames we are receiving */
        frame_list recvd_frames;        /* frames that are completed and ready to be received by the application
                                         * note that this frame_list does not use the last_frame_num and first_frame_num vars of the frame list struct */
                                         
        path_mtu pmtu;                  /* path mtu search values */
        char is_df_set;                 /* boolean switch to indicate if DF bit set in IP header */

        /* time outs (set/read via vfer_setsockopt/getsockopt) */
        int connect_timeout;            /* microseconds ; timeout for vfer_accept() call */
        int close_timeout;              /* microseconds ; timeout for vfer_close() call */
        int accept_timeout;             /* microseconds ; timeout for vfer_accept() call */
        int snd_timeout;                /* microseconds ; timeout for vfer_send() call */
        int rcv_timeout;                /* microseconds ; timeout for vfer_recv() call */

        int backlog;                            /* number of incoming connection to keep in the backlog */
        struct accept_queue* accept_queue;      /* backlog of connections size specified by vfer_listen() */
        int connect_tries;                      /* counter of number of connect tries attempted on this socket */
        int response_tries;                     /* number of times that we've sent a response try on this socket (accepting socket)  */
        struct timeval last_request;            /* timestamp of last request packet sent on this socket */
        struct timeval last_response;           /* timestamp of last response packet sent on this socket */
        struct packet* request;                 /* cached request packet */
        struct packet* response;                /* cached response packet */
        struct vfer_sock* listening_sock;       /* the sock from which we will grab a connection if we are in CONN_ACCEPTING state */
        struct queued_conn* qc;                 /* queued connection which we are currently trying to connect with */
        struct sockaddr_in sa;                  /* used to specify new bound addr on a socket that is in conn_accepting state */
} vfer_sock;


#define MAX_SOCKETS 64
/**
 * @brief a sockets array with a protecting mutex, underlying fd sets, and more.
 */
typedef struct vfer_socks {
        pthread_mutex_t mutex;          /* access to all variables except for the socks array is controlled by this mutex */
        int num_active;                 /* number of active sockets in the list */
        fd_set fds;                     /* fds that are currently active */
        int max_fd;                     /* number of fds in the fds set */
        vfer_sock s[MAX_SOCKETS];       /* static vfer sockets array */
        pthread_mutex_t select_mutex;   /* used to control access to select_cond */
        pthread_cond_t select_cond;     /* condition var used by vfer_select to wait for event notification from ControlT thread */
} vfer_socks_t;

typedef int vfer_fd;

/*
 * variables
 */

vfer_socks_t sockets;   /* all VFER sockets, vfer_fd_t used to index into this array */
long int ReqMem;        /* memory allocated in bytes */

#endif /* VFER_H */

---