lukechilds
/
node
mirror of https://github.com/lukechilds/node.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Upgrade libeio, increase xthread stacksize to 64kb. 

64kb seems to be the magic number for getaddrinfo() to work on Macintosh.
v0.7.4-release
Ryan 16 years ago
parent
342da69701
commit
1df6d61208
5 changed files with 437 additions and 51 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      deps/libeio/Changes
  2. 391
      deps/libeio/eio.c
  3. 78
      deps/libeio/eio.h
  4. 10
      deps/libeio/xthread.h
  5. 2
      wscript

7
deps/libeio/Changes
View File

@ -3,6 +3,13 @@ Revision history for libeio
TODO: maybe add mincore support? available on at leats darwin, solaris, linux, freebsd
1.0
- readdir: correctly handle malloc failures.
- readdir: new flags argument, can return inode
and possibly filetype, can sort in various ways.
- readdir: stop immediately when cancelled, do
not continue reading the directory.
- fix return value of eio_sendfile_sync.
- include sys/mman.h for msync.
- added EIO_STACKSIZE.
- added msync, mtouch support (untested).
- added sync_file_range (untested).

391
deps/libeio/eio.c
View File

@ -1,7 +1,7 @@
/*
* libeio implementation
*
* Copyright (c) 2007,2008 Marc Alexander Lehmann <libeio@schmorp.de>
* Copyright (c) 2007,2008,2009 Marc Alexander Lehmann <libeio@schmorp.de>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modifica-
@ -70,17 +70,35 @@
#ifdef _WIN32
/*doh*/
#else
# include "config.h"
# include <sys/time.h>
# include <sys/select.h>
# include <sys/mman.h>
# include <unistd.h>
# include <utime.h>
# include <signal.h>
# include <dirent.h>
/* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */
# if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)
# define _DIRENT_HAVE_D_TYPE /* sigh */
# define D_INO(de) (de)->d_fileno
# define D_NAMLEN(de) (de)->d_namlen
# elif defined(__linux) || defined(d_ino) || _XOPEN_SOURCE >= 600
# define D_INO(de) (de)->d_ino
# endif
#ifdef _D_EXACT_NAMLEN
# undef D_NAMLEN
# define D_NAMLEN(de) _D_EXACT_NAMLEN (de)
#endif
# ifdef _DIRENT_HAVE_D_TYPE
# define D_TYPE(de) (de)->d_type
# endif
# ifndef EIO_STRUCT_DIRENT
# define EIO_STRUCT_DIRENT struct dirent
# endif
@ -102,6 +120,16 @@
# endif
#endif
#ifndef D_TYPE
# define D_TYPE(de) 0
#endif
#ifndef D_INO
# define D_INO(de) 0
#endif
#ifndef D_NAMLEN
# define D_NAMLEN(de) strlen ((de)->d_name)
#endif
/* number of seconds after which an idle threads exit */
#define IDLE_TIMEOUT 10
@ -161,6 +189,7 @@ static void eio_execute (struct etp_worker *self, eio_req *req);
closedir (wrk->dirp); \
wrk->dirp = 0; \
}
#define ETP_WORKER_COMMON \
void *dbuf; \
DIR *dirp;
@ -773,7 +802,7 @@ eio__pwrite (int fd, void *buf, size_t count, off_t offset)
ooffset = lseek (fd, 0, SEEK_CUR);
lseek (fd, offset, SEEK_SET);
res = write (fd, buf, count);
lseek (fd, offset, SEEK_SET);
lseek (fd, ooffset, SEEK_SET);
X_UNLOCK (preadwritelock);
return res;
@ -840,7 +869,7 @@ eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags)
#endif
/* even though we could play tricks with the flags, it's better to always
* call fdatasync, as thta matches the expectation of it's users best */
* call fdatasync, as that matches the expectation of its users best */
return fdatasync (fd);
}
@ -963,6 +992,169 @@ eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self)
return res;
}
static signed char
eio_dent_cmp (const eio_dirent *a, const eio_dirent *b)
{
return a->score - b->score ? a->score - b->score /* works because our signed char is always 0..100 */
: a->inode < b->inode ? -1 : a->inode > b->inode ? 1 : 0;
}
#define EIO_DENT_CMP(i,op,j) eio_dent_cmp (&i, &j) op 0
#define EIO_SORT_CUTOFF 30 /* quite high, but performs well on many filesystems */
#define EIO_SORT_FAST 60 /* when to only use insertion sort */
static void
eio_dent_radix_sort (eio_dirent *dents, int size, signed char score_bits, ino_t inode_bits)
{
unsigned char bits [9 + sizeof (ino_t) * 8];
unsigned char *bit = bits;
assert (CHAR_BIT == 8);
assert (sizeof (eio_dirent) * 8 < 256);
assert (offsetof (eio_dirent, inode)); /* we use 0 as sentinel */
assert (offsetof (eio_dirent, score)); /* we use 0 as sentinel */
if (size <= EIO_SORT_FAST)
return;
/* first prepare an array of bits to test in our radix sort */
/* try to take endianness into account, as well as differences in ino_t sizes */
/* inode_bits must contain all inodes ORed together */
/* which is used to skip bits that are 0 everywhere, which is very common */
{
ino_t endianness;
int i, j;
/* we store the byte offset of byte n into byte n of "endianness" */
for (i = 0; i < sizeof (ino_t); ++i)
((unsigned char *)&endianness)[i] = i;
*bit++ = 0;
for (i = 0; i < sizeof (ino_t); ++i)
{
/* shifting off the byte offsets out of "endianness" */
int offs = (offsetof (eio_dirent, inode) + (endianness & 0xff)) * 8;
endianness >>= 8;
for (j = 0; j < 8; ++j)
if (inode_bits & (((ino_t)1) << (i * 8 + j)))
*bit++ = offs + j;
}
for (j = 0; j < 8; ++j)
if (score_bits & (1 << j))
*bit++ = offsetof (eio_dirent, score) * 8 + j;
}
/* now actually do the sorting (a variant of MSD radix sort) */
{
eio_dirent *base_stk [9 + sizeof (ino_t) * 8], *base;
eio_dirent *end_stk [9 + sizeof (ino_t) * 8], *end;
unsigned char *bit_stk [9 + sizeof (ino_t) * 8];
int stk_idx = 0;
base_stk [stk_idx] = dents;
end_stk [stk_idx] = dents + size;
bit_stk [stk_idx] = bit - 1;
do
{
base = base_stk [stk_idx];
end = end_stk [stk_idx];
bit = bit_stk [stk_idx];
for (;;)
{
unsigned char O = *bit >> 3;
unsigned char M = 1 << (*bit & 7);
eio_dirent *a = base;
eio_dirent *b = end;
if (b - a < EIO_SORT_CUTOFF)
break;
/* now bit-partition the array on the bit */
/* this ugly asymmetric loop seems to perform much better than typical */
/* partition algos found in the literature */
do
if (!(((unsigned char *)a)[O] & M))
++a;
else if (!(((unsigned char *)--b)[O] & M))
{
eio_dirent tmp = *a; *a = *b; *b = tmp;
++a;
}
while (b > a);
/* next bit, or stop, if no bits left in this path */
if (!*--bit)
break;
base_stk [stk_idx] = a;
end_stk [stk_idx] = end;
bit_stk [stk_idx] = bit;
++stk_idx;
end = a;
}
}
while (stk_idx--);
}
}
static void
eio_dent_insertion_sort (eio_dirent *dents, int size)
{
/* first move the smallest element to the front, to act as a sentinel */
{
int i;
eio_dirent *min = dents;
/* the radix pre-pass ensures that the minimum element is in the first EIO_SORT_CUTOFF + 1 elements */
for (i = size > EIO_SORT_FAST ? EIO_SORT_CUTOFF + 1 : size; --i; )
if (EIO_DENT_CMP (dents [i], <, *min))
min = &dents [i];
/* swap elements 0 and j (minimum) */
{
eio_dirent tmp = *dents; *dents = *min; *min = tmp;
}
}
/* then do standard insertion sort, assuming that all elements are >= dents [0] */
{
eio_dirent *i, *j;
for (i = dents + 1; i < dents + size; ++i)
{
eio_dirent value = *i;
for (j = i - 1; EIO_DENT_CMP (*j, >, value); --j)
j [1] = j [0];
j [1] = value;
}
}
}
static void
eio_dent_sort (eio_dirent *dents, int size, signed char score_bits, ino_t inode_bits)
{
if (size <= 1)
return; /* our insertion sort relies on size > 0 */
/* first we use a radix sort, but only for dirs >= EIO_SORT_FAST */
/* and stop sorting when the partitions are <= EIO_SORT_CUTOFF */
eio_dent_radix_sort (dents, size, score_bits, inode_bits);
/* use an insertion sort at the end, or for small arrays, */
/* as insertion sort is more efficient for small partitions */
eio_dent_insertion_sort (dents, size);
}
/* read a full directory */
static void
eio__scandir (eio_req *req, etp_worker *self)
@ -970,54 +1162,196 @@ eio__scandir (eio_req *req, etp_worker *self)
DIR *dirp;
EIO_STRUCT_DIRENT *entp;
char *name, *names;
int memlen = 4096;
int memofs = 0;
int res = 0;
int namesalloc = 4096;
int namesoffs = 0;
int flags = req->int1;
eio_dirent *dents = 0;
int dentalloc = 128;
int dentoffs = 0;
ino_t inode_bits = 0;
req->result = -1;
if (!(flags & EIO_READDIR_DENTS))
flags &= ~(EIO_READDIR_DIRS_FIRST | EIO_READDIR_STAT_ORDER);
X_LOCK (wrklock);
/* the corresponding closedir is in ETP_WORKER_CLEAR */
self->dirp = dirp = opendir (req->ptr1);
req->flags |= EIO_FLAG_PTR2_FREE;
req->ptr2 = names = malloc (memlen);
req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE;
req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0;
req->ptr2 = names = malloc (namesalloc);
X_UNLOCK (wrklock);
if (dirp && names)
if (dirp && names && (!flags || dents))
for (;;)
{
errno = 0;
entp = readdir (dirp);
if (!entp)
break;
{
if (errno)
break;
/* sort etc. */
req->int1 = flags;
req->result = dentoffs;
if (flags & EIO_READDIR_STAT_ORDER)
eio_dent_sort (dents, dentoffs, 0, inode_bits); /* sort by inode exclusively */
else if (flags & EIO_READDIR_DIRS_FIRST)
if (flags & EIO_READDIR_FOUND_UNKNOWN)
eio_dent_sort (dents, dentoffs, 7, inode_bits); /* sort by score and inode */
else
{
/* in this case, all is known, and we just put dirs first and sort them */
eio_dirent *oth = dents + dentoffs;
eio_dirent *dir = dents;
/* now partition dirs to the front, and non-dirs to the back */
/* by walking from both sides and swapping if necessary */
/* also clear score, so it doesn't influence sorting */
while (oth > dir)
{
if (dir->type == EIO_DT_DIR)
++dir;
else if ((--oth)->type == EIO_DT_DIR)
{
eio_dirent tmp = *dir; *dir = *oth; *oth = tmp;
++dir;
}
}
/* now sort the dirs only */
eio_dent_sort (dents, dir - dents, 0, inode_bits);
}
break;
}
/* now add the entry to our list(s) */
name = entp->d_name;
/* skip . and .. entries */
if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2])))
{
int len = strlen (name) + 1;
res++;
int len = D_NAMLEN (entp) + 1;
while (memofs + len > memlen)
while (expect_false (namesoffs + len > namesalloc))
{
memlen *= 2;
namesalloc *= 2;
X_LOCK (wrklock);
req->ptr2 = names = realloc (names, memlen);
req->ptr2 = names = realloc (names, namesalloc);
X_UNLOCK (wrklock);
if (!names)
break;
}
memcpy (names + memofs, name, len);
memofs += len;
}
}
memcpy (names + namesoffs, name, len);
if (errno)
res = -1;
if (dents)
{
struct eio_dirent *ent;
if (expect_false (dentoffs == dentalloc))
{
dentalloc *= 2;
X_LOCK (wrklock);
req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent));
X_UNLOCK (wrklock);
if (!dents)
break;
}
ent = dents + dentoffs;
ent->nameofs = namesoffs; /* rather dirtily we store the offset in the pointer */
ent->namelen = len - 1;
ent->inode = D_INO (entp);
inode_bits |= ent->inode;
switch (D_TYPE (entp))
{
default:
ent->type = EIO_DT_UNKNOWN;
flags |= EIO_READDIR_FOUND_UNKNOWN;
break;
#ifdef DT_FIFO
case DT_FIFO: ent->type = EIO_DT_FIFO; break;
#endif
#ifdef DT_CHR
case DT_CHR: ent->type = EIO_DT_CHR; break;
#endif
#ifdef DT_MPC
case DT_MPC: ent->type = EIO_DT_MPC; break;
#endif
#ifdef DT_DIR
case DT_DIR: ent->type = EIO_DT_DIR; break;
#endif
#ifdef DT_NAM
case DT_NAM: ent->type = EIO_DT_NAM; break;
#endif
#ifdef DT_BLK
case DT_BLK: ent->type = EIO_DT_BLK; break;
#endif
#ifdef DT_MPB
case DT_MPB: ent->type = EIO_DT_MPB; break;
#endif
#ifdef DT_REG
case DT_REG: ent->type = EIO_DT_REG; break;
#endif
#ifdef DT_NWK
case DT_NWK: ent->type = EIO_DT_NWK; break;
#endif
#ifdef DT_CMP
case DT_CMP: ent->type = EIO_DT_CMP; break;
#endif
#ifdef DT_LNK
case DT_LNK: ent->type = EIO_DT_LNK; break;
#endif
#ifdef DT_SOCK
case DT_SOCK: ent->type = EIO_DT_SOCK; break;
#endif
#ifdef DT_DOOR
case DT_DOOR: ent->type = EIO_DT_DOOR; break;
#endif
#ifdef DT_WHT
case DT_WHT: ent->type = EIO_DT_WHT; break;
#endif
}
ent->score = 7;
if (flags & EIO_READDIR_DIRS_FIRST)
{
if (ent->type == EIO_DT_UNKNOWN)
{
if (*name == '.') /* leading dots are likely directories, and, in any case, rare */
ent->score = 1;
else if (!strchr (name, '.')) /* absense of dots indicate likely dirs */
ent->score = len <= 2 ? 4 - len : len <= 4 ? 4 : len <= 7 ? 5 : 6; /* shorter == more likely dir, but avoid too many classes */
}
else if (ent->type == EIO_DT_DIR)
ent->score = 0;
}
}
req->result = res;
namesoffs += len;
++dentoffs;
}
if (EIO_CANCELLED (req))
{
errno = ECANCELED;
break;
}
}
}
#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
@ -1447,9 +1781,9 @@ eio_req *eio_rmdir (const char *path, int pri, eio_cb cb, void *data)
return eio__1path (EIO_RMDIR, path, pri, cb, data);
}
eio_req *eio_readdir (const char *path, int pri, eio_cb cb, void *data)
eio_req *eio_readdir (const char *path, int flags, int pri, eio_cb cb, void *data)
{
return eio__1path (EIO_READDIR, path, pri, cb, data);
REQ (EIO_READDIR); PATH; req->int1 = flags; SEND;
}
eio_req *eio_mknod (const char *path, mode_t mode, dev_t dev, int pri, eio_cb cb, void *data)
@ -1548,12 +1882,15 @@ void eio_grp_add (eio_req *grp, eio_req *req)
ssize_t eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count)
{
etp_worker wrk;
ssize_t ret;
wrk.dbuf = 0;
eio__sendfile (ofd, ifd, offset, count, &wrk);
ret = eio__sendfile (ofd, ifd, offset, count, &wrk);
if (wrk.dbuf)
free (wrk.dbuf);
return ret;
}

78
deps/libeio/eio.h
View File

@ -1,7 +1,7 @@
/*
* libeio API header
*
* Copyright (c) 2007,2008 Marc Alexander Lehmann <libeio@schmorp.de>
* Copyright (c) 2007,2008,2009 Marc Alexander Lehmann <libeio@schmorp.de>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modifica-
@ -47,7 +47,8 @@ extern "C" {
#include <stddef.h>
#include <sys/types.h>
typedef struct eio_req eio_req;
typedef struct eio_req eio_req;
typedef struct eio_dirent eio_dirent;
typedef int (*eio_cb)(eio_req *req);
@ -59,6 +60,60 @@ typedef int (*eio_cb)(eio_req *req);
# define EIO_STRUCT_STAT struct stat
#endif
/* for readdir */
/* eio_readdir flags */
enum {
EIO_READDIR_DENTS = 0x01, /* ptr2 contains eio_dirents, not just the (unsorted) names */
EIO_READDIR_DIRS_FIRST = 0x02, /* dirents gets sorted into a good stat() ing order to find directories first */
EIO_READDIR_STAT_ORDER = 0x04, /* dirents gets sorted into a good stat() ing order to quickly stat all files */
EIO_READDIR_FOUND_UNKNOWN = 0x80, /* set by eio_readdir when *_ARRAY was set and any TYPE=UNKNOWN's were found */
EIO_READDIR_CUSTOM1 = 0x100, /* for use by apps */
EIO_READDIR_CUSTOM2 = 0x200 /* for use by apps */
};
/* using "typical" values in the hope that the compiler will do something sensible */
enum eio_dtype {
EIO_DT_UNKNOWN = 0,
EIO_DT_FIFO = 1,
EIO_DT_CHR = 2,
EIO_DT_MPC = 3, /* multiplexed char device (v7+coherent) */
EIO_DT_DIR = 4,
EIO_DT_NAM = 5, /* xenix special named file */
EIO_DT_BLK = 6,
EIO_DT_MPB = 7, /* multiplexed block device (v7+coherent) */
EIO_DT_REG = 8,
EIO_DT_NWK = 9, /* HP-UX network special */
EIO_DT_CMP = 9, /* VxFS compressed */
EIO_DT_LNK = 10,
/* DT_SHAD = 11,*/
EIO_DT_SOCK = 12,
EIO_DT_DOOR = 13, /* solaris door */
EIO_DT_WHT = 14,
EIO_DT_MAX = 15 /* highest DT_VALUE ever, hopefully */
};
struct eio_dirent {
int nameofs; /* offset of null-terminated name string in (char *)req->ptr2 */
unsigned short namelen; /* size of filename without trailing 0 */
unsigned char type; /* one of EIO_DT_* */
signed char score; /* internal use */
ino_t inode; /* the inode number, if available, otherwise unspecified */
};
/* eio_sync_file_range flags */
enum {
EIO_SYNC_FILE_RANGE_WAIT_BEFORE = 1,
EIO_SYNC_FILE_RANGE_WRITE = 2,
EIO_SYNC_FILE_RANGE_WAIT_AFTER = 4
};
typedef double eio_tstamp; /* feel free to use double in your code directly */
/* the eio request structure */
enum {
EIO_CUSTOM,
EIO_OPEN, EIO_CLOSE, EIO_DUP2,
@ -78,18 +133,9 @@ enum {
EIO_BUSY
};
/* eio_sync_file_range flags */
enum {
EIO_SYNC_FILE_RANGE_WAIT_BEFORE = 1,
EIO_SYNC_FILE_RANGE_WRITE = 2,
EIO_SYNC_FILE_RANGE_WAIT_AFTER = 4
};
typedef double eio_tstamp; /* feel free to use double in your code directly */
/* eio request structure */
/* this structure is mostly read-only */
/* when initialising it, all members must be zero-initialised */
struct eio_req
{
eio_req volatile *next; /* private ETP */
@ -97,13 +143,13 @@ struct eio_req
ssize_t result; /* result of syscall, e.g. result = read (... */
off_t offs; /* read, write, truncate, readahead, sync_file_range: file offset */
size_t size; /* read, write, readahead, sendfile, msync, sync_file_range: length */
void *ptr1; /* all applicable requests: pathname, old name */
void *ptr2; /* all applicable requests: new name or memory buffer */
void *ptr1; /* all applicable requests: pathname, old name; readdir: optional eio_dirents */
void *ptr2; /* all applicable requests: new name or memory buffer; readdir: name strings */
eio_tstamp nv1; /* utime, futime: atime; busy: sleep time */
eio_tstamp nv2; /* utime, futime: mtime */
int type; /* EIO_xxx constant ETP */
int int1; /* all applicable requests: file descriptor; sendfile: output fd; open, msync: flags */
int int1; /* all applicable requests: file descriptor; sendfile: output fd; open, msync, readdir: flags */
long int2; /* chown, fchown: uid; sendfile: input fd; open, chmod, mkdir, mknod: file mode, sync_file_range: flags */
long int3; /* chown, fchown: gid; mknod: dev_t */
int errorno; /* errno value on syscall return */
@ -192,7 +238,7 @@ eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void
eio_req *eio_chown (const char *path, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data);
eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data);
eio_req *eio_mkdir (const char *path, mode_t mode, int pri, eio_cb cb, void *data);
eio_req *eio_readdir (const char *path, int pri, eio_cb cb, void *data); /* result=ptr2 allocated dynamically */
eio_req *eio_readdir (const char *path, int flags, int pri, eio_cb cb, void *data); /* result=ptr2 allocated dynamically */
eio_req *eio_rmdir (const char *path, int pri, eio_cb cb, void *data);
eio_req *eio_unlink (const char *path, int pri, eio_cb cb, void *data);
eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data); /* result=ptr2 allocated dynamically */

10
deps/libeio/xthread.h
View File

@ -118,8 +118,8 @@ typedef pthread_t thread_t;
# define PTHREAD_STACK_MIN 0
#endif
#ifndef XTHREAD_STACKSIZE
# define XTHREAD_STACKSIZE sizeof (long) * 4096
#ifndef X_STACKSIZE
# define X_STACKSIZE sizeof (long) * 4096
#endif
static int
@ -131,11 +131,7 @@ thread_create (thread_t *tid, void *(*proc)(void *), void *arg)
pthread_attr_init (&attr);
pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
if (XTHREAD_STACKSIZE > 0)
pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN > (XTHREAD_STACKSIZE)
? PTHREAD_STACK_MIN : (XTHREAD_STACKSIZE));
pthread_attr_setstacksize (&attr, PTHREAD_STACK_MIN < X_STACKSIZE ? X_STACKSIZE : PTHREAD_STACK_MIN);
#ifdef PTHREAD_SCOPE_PROCESS
pthread_attr_setscope (&attr, PTHREAD_SCOPE_PROCESS);
#endif

2
wscript
View File

@ -88,7 +88,7 @@ def configure(conf):
conf.define("HAVE_CONFIG_H", 1)
conf.env.append_value("CCFLAGS", "-DEIO_STACKSIZE=%d" % (4096*8))
conf.env.append_value("CCFLAGS", "-DX_STACKSIZE=%d" % (1024*64))
# Split off debug variant before adding variant specific defines
debug_env = conf.env.copy()

Write Preview
Loading…
Cancel
Save