Geek asks and answers

Python walk but threads

I would like to recursively traverse a directory, but I want python to interrupt from any single listdir if it encounters a directory with more than 100 files. Basically, I am looking for a file (.TXT), but I want to avoid directories with large sequences of DPX images (usually 10,000 files). Since DPX live in directories on their own without subdirectories, I would like to break this loop as soon as possible.

In short, if python encounters a file matching ".DPX $", it stops listing the subdirectory, returns, skips that subdirectory, and continues to walk in other subdirectories.

Is it possible to break the directory list loop before all list results are returned?

+5
source share
3 answers

os.listdir - os, @Charles Duffy.

: ,

, .

from ctypes import CDLL, c_char_p, c_int, c_long, c_ushort, c_byte, c_char, Structure, POINTER, byref, cast, sizeof, get_errno
from ctypes.util import find_library

class c_dir(Structure):
    """Opaque type for directory entries, corresponds to struct DIR"""
    pass

class c_dirent(Structure):
    """Directory entry"""
    # FIXME not sure these are the exactly correct types!
    _fields_ = (
        ('d_ino', c_long), # inode number
        ('d_off', c_long), # offset to the next dirent
        ('d_reclen', c_ushort), # length of this record
        ('d_type', c_byte), # type of file; not supported by all file system types
        ('d_name', c_char * 4096) # filename
        )
c_dirent_p = POINTER(c_dirent)
c_dirent_pp = POINTER(c_dirent_p)
c_dir_p = POINTER(c_dir)

c_lib = CDLL(find_library("c"))
opendir = c_lib.opendir
opendir.argtypes = [c_char_p]
opendir.restype = c_dir_p

readdir_r = c_lib.readdir_r
readdir_r.argtypes = [c_dir_p, c_dirent_p, c_dirent_pp]
readdir_r.restype = c_int

closedir = c_lib.closedir
closedir.argtypes = [c_dir_p]
closedir.restype = c_int

import errno

def listdirx(path):
    """
    A generator to return the names of files in the directory passed in
    """
    dir_p = opendir(path)

    if not dir_p:
        raise IOError()

    entry_p = cast(c_lib.malloc(sizeof(c_dirent)), c_dirent_p)

    try:
        while True:
            res = readdir_r(dir_p, entry_p, byref(entry_p))
            if res:
                raise IOError()
            if not entry_p:
                break
            name = entry_p.contents.d_name
            if name not in (".", ".."):
                yield name
    finally:
        if dir_p:
            closedir(dir_p)
        if entry_p:
            c_lib.free(entry_p)

if __name__ == '__main__':
    import sys
    path = sys.argv[1]
    max_per_dir = int(sys.argv[2])
    for idx, entry in enumerate(listdirx(path)):
        if idx >= max_per_dir:
            break
        print entry
+1

" " os.listdir(), . . os.path.walk() os.walk() , DPX. os.walk() , , Python, . os.path.walk() , .

+4

os.walk:

downdown True, dirnames (, del ), walk() , dirnames; . dirnames, topdown False , dirnames , dirnames .

, dirnames, os.walk . "... del slice assign"; dirnames=[], dirnames.

+2

More articles:


All Articles