Subject: chown|chgrp|chmod -R much too slow +FIX (#195) Index: bin,etc/chown.c,chgrp.c,chmod.c 2.11BSD Description: The recursive forms of the 'chown', 'chgrp' and 'chmod' commands are extremely slow when used on directory trees with more than a small number of subdirectories. Repeat-By: chmod -R a+r /usr/src Just kidding, sort of. For something more system friendly create a test directory tree, something like this will do: mkdir -p /tmp/src mkdir -p /tmp/src/sys mkdir -p /tmp/src/sys/pdpuba mkdir -p /tmp/src/sys/bootrom mkdir -p /tmp/src/sys/conf mkdir -p /tmp/src/sys/conf/boot mkdir -p /tmp/src/sys/conf/VAX.compile mkdir -p /tmp/src/sys/conf/VAX.compile/cpp mkdir -p /tmp/src/sys/conf/spl mkdir -p /tmp/src/sys/conf/spl_3com mkdir -p /tmp/src/sys/sys mkdir -p /tmp/src/sys/net mkdir -p /tmp/src/sys/h mkdir -p /tmp/src/sys/netimp mkdir -p /tmp/src/sys/autoconfig mkdir -p /tmp/src/sys/netns mkdir -p /tmp/src/sys/pdp mkdir -p /tmp/src/sys/mdec mkdir -p /tmp/src/sys/netinet mkdir -p /tmp/src/sys/pdpstand mkdir -p /tmp/src/sys/pdpstand/NEW mkdir -p /tmp/src/sys/pdpstand/OLD mkdir -p /tmp/src/sys/OTHERS mkdir -p /tmp/src/sys/OTHERS/dm11 mkdir -p /tmp/src/sys/OTHERS/cat mkdir -p /tmp/src/sys/OTHERS/rs03.04 mkdir -p /tmp/src/sys/OTHERS/dvhp mkdir -p /tmp/src/sys/OTHERS/rp03 mkdir -p /tmp/src/sys/OTHERS/cr11 Then: time chmod -R a+r /tmp/src On an 11/73 the results were: 0.2u 2.9s 0:05 54% 128+7io After applying this update the results are: 0.0u 0.8s 0:01 56% 38+8io Fix: The slowness of the recursive forms of these commands is totally due to use of 'getwd'. Each time a subdirectory is entered the program does a 'getwd()' call to save the current directory it is in. What I have done is make use of the 'fchdir()' syscall (added in update #187). Since chmod and friends have the current directory open (after all, an 'opendir(".")' was done so as to scan the directory) why not simply pass the file descriptor as an argument when recursing? Instead of doing a 'getwd()' on entry (to the recursive function) and a 'chdir()' on exit only a 'fchdir()' need be done on exit. The speed increase is amazing and as an added benefit the amount of memory used is dramatically reduced. Previously the call to 'getwd' required "char savedir[1024];" for each subdirectory level entered. Now only an 'int' is allocated on the stack. One minor change was needed outside the recursive function. In the 'main()' routine the current directory ('.') is opened and passed in as the known starting point. Five files are modified by this update: /usr/src/sys/h/dir.h /usr/src/bin/chmod.c /usr/src/bin/chgrp.c /usr/src/etc/chown.c /VERSION In dir.h a new macro is defined (borrowed from 4.4). 'dirfd' extracts the file descriptor from a DIR structure. The change to /VERSION is _long_ overdue. The current update level of the system will now be updated in /VERSION each time a change is posted. To apply this update: 1) save the patch to a file, /tmp/p 2) patch -p < /tmp/p 3) cd /usr/src/etc make chown install -s -m 755 chown /etc 4) cd /usr/src/bin make chmod chgrp install -s -m 755 chgrp chmod /bin To give an idea of the speed increase when traversing large directory trees here are the timings for changing the mode on 456 subdirectories (i replicated the /usr/src directories into /tmp - the actual data files were not copied over): time /bin/chmod.old -R a+r /tmp/src 6.4u 159.1s 3:30 78% 1572+70io 0ov 0sw time /bin/chmod -R a+r /tmp/src 0.5u 15.3s 0:22 69% 528+57io 0ov 0sw I'd say that was a "modest" improvement indeed. As always the 2.11BSD updates are available via anonymous FTP at ftp.iipo.gtegsc.com in the directory /pub/2.11BSD ======================cut here *** /usr/src/sys/h/dir.h.old Sat May 26 00:09:35 1990 --- /usr/src/sys/h/dir.h Fri Nov 4 20:28:16 1994 *************** *** 3,9 **** * All rights reserved. The Berkeley software License Agreement * specifies the terms and conditions for redistribution. * ! * @(#)dir.h 1.1 (2.10BSD Berkeley) 12/1/86 */ #ifndef _DIR_ --- 3,9 ---- * All rights reserved. The Berkeley software License Agreement * specifies the terms and conditions for redistribution. * ! * @(#)dir.h 1.2 (2.11BSD GTE) 11/4/94 */ #ifndef _DIR_ *************** *** 69,74 **** --- 69,75 ---- char dd_buf[DIRBLKSIZ]; struct direct dd_cur; } DIR; + #ifndef NULL #define NULL 0 #endif *************** *** 80,85 **** --- 81,87 ---- extern long telldir(); extern void seekdir(); #define rewinddir(dirp) seekdir((dirp), (long)0) + #define dirfd(dirp) ((dirp)->dd_fd) extern void closedir(); #endif KERNEL *** /usr/src/bin/chmod.c.old Sun Feb 8 14:24:53 1987 --- /usr/src/bin/chmod.c Fri Nov 4 21:09:01 1994 *************** *** 4,11 **** * specifies the terms and conditions for redistribution. */ ! #ifndef lint ! static char sccsid[] = "@(#)chmod.c 5.5 (Berkeley) 5/22/86"; #endif /* --- 4,11 ---- * specifies the terms and conditions for redistribution. */ ! #if !defined(lint) && defined(DOSCCS) ! static char sccsid[] = "@(#)chmod.c 5.5.1 (2.11BSD GTE) 11/4/94"; #endif /* *************** *** 16,24 **** --- 16,26 ---- */ #include #include + #include #include #include + static char *fchdirmsg = "Can't fchdir() back to starting directory"; char *modestring, *ms; int um; int status; *************** *** 31,36 **** --- 33,39 ---- register char *p, *flags; register int i; struct stat st; + int fcurdir; if (argc < 3) { fprintf(stderr, *************** *** 58,63 **** --- 61,73 ---- modestring = argv[0]; um = umask(0); (void) newmode(0); + if (rflag) + { + fcurdir = open(".", O_RDONLY); + if (fcurdir < 0) + fatal(255, "Can't open ."); + } + for (i = 1; i < argc; i++) { p = argv[i]; /* do stat for directory arguments */ *************** *** 66,72 **** continue; } if (rflag && (st.st_mode&S_IFMT) == S_IFDIR) { ! status += chmodr(p, newmode(st.st_mode)); continue; } if ((st.st_mode&S_IFMT) == S_IFLNK && stat(p, &st) < 0) { --- 76,82 ---- continue; } if (rflag && (st.st_mode&S_IFMT) == S_IFDIR) { ! status += chmodr(p, newmode(st.st_mode), fcurdir); continue; } if ((st.st_mode&S_IFMT) == S_IFLNK && stat(p, &st) < 0) { *************** *** 78,97 **** continue; } } exit(status); } ! chmodr(dir, mode) char *dir; { register DIR *dirp; register struct direct *dp; ! register struct stat st; ! char savedir[1024]; int ecode; - if (getwd(savedir) == 0) - fatal(255, "%s", savedir); /* * Change what we are given before doing it's contents */ --- 88,107 ---- continue; } } + close(fcurdir); exit(status); } ! chmodr(dir, mode, savedir) char *dir; + int mode; + int savedir; { register DIR *dirp; register struct direct *dp; ! struct stat st; int ecode; /* * Change what we are given before doing it's contents */ *************** *** 116,122 **** continue; } if ((st.st_mode&S_IFMT) == S_IFDIR) { ! ecode = chmodr(dp->d_name, newmode(st.st_mode)); if (ecode) break; continue; --- 126,132 ---- continue; } if ((st.st_mode&S_IFMT) == S_IFDIR) { ! ecode = chmodr(dp->d_name, newmode(st.st_mode), dirfd(dirp)); if (ecode) break; continue; *************** *** 127,135 **** (ecode = Perror(dp->d_name))) break; } closedir(dirp); - if (chdir(savedir) < 0) - fatal(255, "can't change back to %s", savedir); return (ecode); } --- 137,145 ---- (ecode = Perror(dp->d_name))) break; } + if (fchdir(savedir) < 0) + fatal(255, fchdirmsg); closedir(dirp); return (ecode); } *** /usr/src/bin/chgrp.c.old Sun Feb 8 14:24:54 1987 --- /usr/src/bin/chgrp.c Fri Nov 4 21:43:13 1994 *************** *** 4,12 **** * specifies the terms and conditions for redistribution. */ ! #ifndef lint ! static char sccsid[] = "@(#)chgrp.c 5.7 (Berkeley) 6/4/86"; ! #endif not lint /* * chgrp -fR gid file ... --- 4,12 ---- * specifies the terms and conditions for redistribution. */ ! #if !defined(lint) && defined(DOSCCS) ! static char sccsid[] = "@(#)chgrp.c 5.7.1 (2.11BSD GTE) 11/4/94"; ! #endif /* * chgrp -fR gid file ... *************** *** 15,20 **** --- 15,21 ---- #include #include #include + #include #include #include #include *************** *** 23,33 **** struct group *gr, *getgrnam(), *getgrgid(); struct passwd *getpwuid(), *pwd; struct stat stbuf; ! int gid, uid; int status; int fflag, rflag; ! /* VARARGS */ ! int fprintf(); main(argc, argv) int argc; --- 24,34 ---- struct group *gr, *getgrnam(), *getgrgid(); struct passwd *getpwuid(), *pwd; struct stat stbuf; ! gid_t gid; ! uid_t uid; int status; int fflag, rflag; ! static char *fchdirmsg = "Can't fchdir() back to starting directory"; main(argc, argv) int argc; *************** *** 35,40 **** --- 36,42 ---- { register c, i; register char *cp; + int fcurdir; argc--, argv++; while (argc > 0 && argv[0][0] == '-') { *************** *** 82,87 **** --- 84,93 ---- fatal(255, "You are not a member of the %s group", argv[0]); } ok: + fcurdir = open(".", O_RDONLY); + if (fcurdir < 0) + fatal(255, "Can't open ."); + for (c = 1; c < argc; c++) { /* do stat for directory arguments */ if (lstat(argv[c], &stbuf)) { *************** *** 93,99 **** continue; } if (rflag && ((stbuf.st_mode & S_IFMT) == S_IFDIR)) { ! status += chownr(argv[c], stbuf.st_uid, gid); continue; } if (chown(argv[c], -1, gid)) { --- 99,105 ---- continue; } if (rflag && ((stbuf.st_mode & S_IFMT) == S_IFDIR)) { ! status += chownr(argv[c], stbuf.st_uid, gid, fcurdir); continue; } if (chown(argv[c], -1, gid)) { *************** *** 115,131 **** return (1); } ! chownr(dir, uid, gid) char *dir; { register DIR *dirp; register struct direct *dp; ! register struct stat st; ! char savedir[1024]; int ecode; - if (getwd(savedir) == 0) - fatal(255, "%s", savedir); /* * Change what we are given before doing its contents. */ --- 121,137 ---- return (1); } ! chownr(dir, uid, gid, savedir) char *dir; + uid_t uid; + gid_t gid; + int savedir; { register DIR *dirp; register struct direct *dp; ! struct stat st; int ecode; /* * Change what we are given before doing its contents. */ *************** *** 155,161 **** continue; } if ((st.st_mode & S_IFMT) == S_IFDIR) { ! ecode = chownr(dp->d_name, st.st_uid, gid); if (ecode) break; continue; --- 161,167 ---- continue; } if ((st.st_mode & S_IFMT) == S_IFDIR) { ! ecode = chownr(dp->d_name, st.st_uid, gid, dirfd(dirp)); if (ecode) break; continue; *************** *** 164,172 **** (ecode = Perror(dp->d_name))) break; } closedir(dirp); - if (chdir(savedir) < 0) - fatal(255, "can't change back to %s", savedir); return (ecode); } --- 170,178 ---- (ecode = Perror(dp->d_name))) break; } + if (fchdir(savedir) < 0) + fatal(255, fchdirmsg); closedir(dirp); return (ecode); } *************** *** 182,187 **** --- 188,194 ---- return (!fflag); } + /* VARARGS */ fatal(status, fmt, a) int status; char *fmt, *a; *** /usr/src/etc/chown.c.old Sun Feb 15 20:51:19 1987 --- /usr/src/etc/chown.c Fri Nov 4 22:14:17 1994 *************** *** 4,17 **** * specifies the terms and conditions for redistribution. */ ! #ifndef lint char copyright[] = "@(#) Copyright (c) 1980 Regents of the University of California.\n\ All rights reserved.\n"; - #endif ! #ifndef lint ! static char sccsid[] = "@(#)chown.c 5.6 (Berkeley) 5/29/86"; #endif /* --- 4,15 ---- * specifies the terms and conditions for redistribution. */ ! #if !defined(lint) && defined(DOSCCS) char copyright[] = "@(#) Copyright (c) 1980 Regents of the University of California.\n\ All rights reserved.\n"; ! static char sccsid[] = "@(#)chown.c 5.6.1 (2.11BSD GTE) 11/4/94"; #endif /* *************** *** 21,26 **** --- 19,25 ---- #include #include #include + #include #include #include #include *************** *** 27,36 **** #include #include struct passwd *pwd; struct passwd *getpwnam(); struct stat stbuf; ! int uid; int status; int fflag; int rflag; --- 26,36 ---- #include #include + static char *fchdirmsg = "Can't fchdir() back to starting directory"; struct passwd *pwd; struct passwd *getpwnam(); struct stat stbuf; ! uid_t uid; int status; int fflag; int rflag; *************** *** 38,46 **** main(argc, argv) char *argv[]; { ! register int c, gid; register char *cp, *group; struct group *grp; argc--, argv++; while (argc > 0 && argv[0][0] == '-') { --- 38,48 ---- main(argc, argv) char *argv[]; { ! register int c; ! register gid_t gid; register char *cp, *group; struct group *grp; + int fcurdir; argc--, argv++; while (argc > 0 && argv[0][0] == '-') { *************** *** 81,86 **** --- 83,93 ---- uid = pwd->pw_uid; } else uid = atoi(argv[0]); + + fcurdir = open(".", O_RDONLY); + if (fcurdir < 0) + fatal(255, "Can't open ."); + for (c = 1; c < argc; c++) { /* do stat for directory arguments */ if (lstat(argv[c], &stbuf) < 0) { *************** *** 88,94 **** continue; } if (rflag && ((stbuf.st_mode&S_IFMT) == S_IFDIR)) { ! status += chownr(argv[c], uid, gid); continue; } if (chown(argv[c], uid, gid)) { --- 95,101 ---- continue; } if (rflag && ((stbuf.st_mode&S_IFMT) == S_IFDIR)) { ! status += chownr(argv[c], uid, gid, fcurdir); continue; } if (chown(argv[c], uid, gid)) { *************** *** 110,127 **** return (1); } ! chownr(dir, uid, gid) char *dir; { register DIR *dirp; register struct direct *dp; struct stat st; - char savedir[1024]; int ecode; - extern char *getwd(); - if (getwd(savedir) == (char *)0) - fatal(255, "%s", savedir); /* * Change what we are given before doing it's contents. */ --- 117,130 ---- return (1); } ! chownr(dir, uid, gid, savedir) char *dir; { register DIR *dirp; register struct direct *dp; struct stat st; int ecode; /* * Change what we are given before doing it's contents. */ *************** *** 146,152 **** continue; } if ((st.st_mode&S_IFMT) == S_IFDIR) { ! ecode = chownr(dp->d_name, uid, gid); if (ecode) break; continue; --- 149,155 ---- continue; } if ((st.st_mode&S_IFMT) == S_IFDIR) { ! ecode = chownr(dp->d_name, uid, gid, dirfd(dirp)); if (ecode) break; continue; *************** *** 155,163 **** (ecode = Perror(dp->d_name))) break; } closedir(dirp); - if (chdir(savedir) < 0) - fatal(255, "can't change back to %s", savedir); return (ecode); } --- 158,166 ---- (ecode = Perror(dp->d_name))) break; } + if (fchdir(savedir) < 0) + fatal(255, fchdirmsg); closedir(dirp); return (ecode); } *** /VERSION.old Wed Dec 19 09:43:21 1990 --- /VERSION Fri Nov 4 23:19:02 1994 *************** *** 1,3 **** --- 1,5 ---- + Current Patch Level: 195 + 2.11 BSD ============ NOTE --