Subject: Re: [Patch] An alternative to ares_fds() and ares_getsock()

Re: [Patch] An alternative to ares_fds() and ares_getsock()

From: William Ahern <william_at_25thandclement.com>
Date: 2006-02-10

On Fri, Feb 10, 2006 at 11:08:42PM +0100, John Bäckstrand wrote:
> Btw, could you (or anyone else?) let me look at any example source where
> you handle the I/O abstraction for c-ares with something other than
> select? I would very much appreciate it.

I'll try to get permission to release the entire library, but here's the
core of how I've managed so far. FYI, the LIST macro's come from
sys/queue.h, found on all the BSD's and (in a neutered form) with Linux.
You'll have to intuit what some of the other structures look like.

/*
 * How do you interface Ares with libevent?
 *
 * Ares provides an interface, ares_fds(), to determine what sockets it is
 * interested in. This takes two fd_sets, in one it sets all the sockets it
 * is waiting to read from, and in the other all the sockets it is waiting
 * to write to. Also, ares_timeout() tells us when Ares wants to be woken to
 * timeout any outstanding requests.
 *
 * ares_catch_event() simply catches each socket event from the libevent
 * dispatcher and notifies Ares--using ares_process()--that a particular
 * socket is ready.
 *
 * ares_reset_events() is the meat of the interface, though. When invoked it
 * calls ares_fds(). While looping over over those sets there is an
 * inner-loop over an internal list of pending events. If there's a match
 * the pending event is put onto the live list. If there's no match then a
 * new event struct is created, set, inserted into the event loop and placed
 * on the live list. At the end of this process, we loop again over our
 * pending list to reap dead Ares events. The live list is then copied over
 * to the pending list.
 *
 * Zeroing and then [possibly] scanning 1K+ fd_sets on every lookup request
 * and I/O readiness event may turn out to be dreadfully slow. It would
 * probably be worth it to 1) teach Ares to deal with struct pollfd's (such
 * as ADNS does) or 2) teach the libevent interface to Ares.
 *
 */
static void ares_catch_event(int fd, short events, void *null); /* Declare early */

static void ares_reset_events(struct ares_channel *c) {
        fd_set rd, wr;
        int fd, max;
        struct ares_event *ev, *nxt;
        int events;
        struct timeval timeout;
        LIST_HEAD(,ares_event) live;
        LIST_HEAD(,ares_event) dead;

        FD_ZERO(&rd);
        FD_ZERO(&wr);

        LIST_INIT(&live);
        LIST_INIT(&dead);

        max = ares_fds(c->channel,&rd,&wr);

        fd = -1;
next_fd:
        for (fd++; fd < max; fd++) {
                events = 0;

                if (FD_ISSET(fd,&rd))
                        events |= EV_READ;
                
                if (FD_ISSET(fd,&wr))
                        events |= EV_WRITE;
                        
                if (!events)
                        continue;

                /* Check to see if we are already doing something w/ the
                 * descriptor. If so modify it appropriately
                 */
                for (ev = LIST_FIRST(&c->events); ev; ev = nxt) {
                        nxt = LIST_NEXT(ev,le);

                        if (ev->fd == fd) {
                                LIST_REMOVE(ev,le);

                                if (ev->events == events) {
                                        LIST_INSERT_HEAD(&live,ev,le);

                                        goto next_fd;
                                } else if (events) {
                                        #if 0
                                        assert(0 == event_del(&ev->ev));
                                        #else
                                        if (0 != event_del(&ev->ev) && errno != EBADF)
                                                warn("event_del");
                                        #endif

                                        LIST_INSERT_HEAD(&dead,ev,le);
                                }

                                break;
                        }
                } /* LIST_FOREACH() */

                /*
                 * Apparently there were no previous events pending for this
                 * descriptor so we'll allocate new event for Ares and
                 * insert into the event loop.
                 */

                ev = LIST_FIRST(&dead);

                if (!ev) {
                        ev = pool_get(lookups.object_pool,sizeof *ev);

                        if (!ev) {
                                warn("unable to allocate event");
                                break;
                                /*
                                 * XXX: If we just silently forget the event
                                 * for now, presumably when memory becomes
                                 * available we'll pick it up.
                                 */
                        }
                } else
                        LIST_REMOVE(ev,le);

                ev->fd = fd;
                ev->events = events;

                event_set(&ev->ev,ev->fd,ev->events | EV_PERSIST,ares_catch_event,c);

                if (0 != event_add(&ev->ev,NULL)) {
                        warn("error adding lookup event");
                        LIST_INSERT_HEAD(&dead,ev,le);
                } else
                        LIST_INSERT_HEAD(&live,ev,le);

        } /* for() */

        for (ev = LIST_FIRST(&c->events); ev; ev = nxt) {
                nxt = LIST_NEXT(ev,le);

                #if 0
                assert(0 == event_del(&ev->ev));
                #else
                if (0 != event_del(&ev->ev) && errno != EBADF)
                        warn("event_del");
                #endif

                LIST_REMOVE(ev,le);
                LIST_INSERT_HEAD(&dead,ev,le);
        }
        /*
         * Anything left on the pending queue was dead.
         */

        for (ev = LIST_FIRST(&live); ev; ev = nxt) {
                nxt = LIST_NEXT(ev,le);
                
                LIST_REMOVE(ev,le);
                LIST_INSERT_HEAD(&c->events,ev,le);
        }
        /*LIST_FIRST(&lookups.ares.events) = LIST_FIRST(&live); <-- Not correct */
        /* Copy live queue to pending queue */

        for (ev = LIST_FIRST(&dead); ev; ev = nxt) {
                nxt = LIST_NEXT(ev,le);
        
                LIST_REMOVE(ev,le);

                pool_put(lookups.object_pool,ev);
        }
        /* Bury dead event struct's */

        timerclear(&timeout);

        (void)ares_timeout(c->channel,NULL,&timeout);
        /* Ask ares what timeout event it wants, but first zero
         * the timeout struct because Ares won't set it to zero
         * if it doesn't want a timeout
         */

        #if 0 /* Make the timer more likely to fire prematurely */
        if (timerisset(&timeout))
                timeout.tv_usec = timeout.tv_sec * 100, timeout.tv_sec = 0;
        #endif

        if (!timercmp(&timeout,&c->timeout,==)) {
                /* Ares wants a different timeout than previously set */

                if (timerisset(&c->timeout)) {
                        /* We have an outstanding timeout. */

                        assert(0 == evtimer_del(&c->timer));

                        timerclear(&c->timeout);

                        /* Removed outstanding timeout */
                }

                if (timerisset(&timeout)) {
                        /* Ares wants a real timeout (not no timeout) */

                        evtimer_set(&c->timer,ares_catch_event,c);

                        assert (0 == evtimer_add(&c->timer,&timeout));

                        c->timeout = timeout;
                }
        }

        return /* void */;
} /* ares_reset_events() */

/*
 * Called from libevent. The event was submitted by ares_reset_events().
 *
 * fd the descriptor with a waiting event
 * events the waiting events. EV_READ, EV_WRITE or EV_TIMEOUT
 * null reset_events() didn't pass a token to event_set()
 */
static void ares_catch_event(int fd, short events, void *arg) {
        struct ares_channel *c = arg;
        fd_set rd, wr;

        FD_ZERO(&rd);
        FD_ZERO(&wr);

        if (events & EV_READ)
                FD_SET(fd,&rd);

        if (events & EV_WRITE)
                FD_SET(fd,&wr);

        if (events & EV_TIMEOUT)
                timerclear(&c->timeout);

        ares_process(c->channel,&rd,&wr);

        ares_reset_events(c);
} /* ares_catch_event() */
Received on Fri Feb 10 23:43:56 2006