#!/usr/bin/env python
# encoding: utf-8
# Thomas Nagy, 2005-2018 (ita)
"""
Node: filesystem structure
#. Each file/folder is represented by exactly one node.
#. Some potential class properties are stored on :py:class:`waflib.Build.BuildContext` : nodes to depend on, etc.
Unused class members can increase the `.wafpickle` file size sensibly.
#. Node objects should never be created directly, use
the methods :py:func:`Node.make_node` or :py:func:`Node.find_node` for the low-level operations
#. The methods :py:func:`Node.find_resource`, :py:func:`Node.find_dir` :py:func:`Node.find_or_declare` must be
used when a build context is present
#. Each instance of :py:class:`waflib.Context.Context` has a unique :py:class:`Node` subclass required for serialization.
(:py:class:`waflib.Node.Nod3`, see the :py:class:`waflib.Context.Context` initializer). A reference to the context
owning a node is held as *self.ctx*
"""
import os, re, sys, shutil
from waflib import Utils, Errors
exclude_regs = '''
**/*~
**/#*#
**/.#*
**/%*%
**/._*
**/*.swp
**/CVS
**/CVS/**
**/.cvsignore
**/SCCS
**/SCCS/**
**/vssver.scc
**/.svn
**/.svn/**
**/BitKeeper
**/.git
**/.git/**
**/.gitignore
**/.bzr
**/.bzrignore
**/.bzr/**
**/.hg
**/.hg/**
**/_MTN
**/_MTN/**
**/.arch-ids
**/{arch}
**/_darcs
**/_darcs/**
**/.intlcache
**/.DS_Store'''
"""
Ant patterns for files and folders to exclude while doing the
recursive traversal in :py:meth:`waflib.Node.Node.ant_glob`
"""
def ant_matcher(s, ignorecase):
reflags = re.I if ignorecase else 0
ret = []
for x in Utils.to_list(s):
x = x.replace('\\', '/').replace('//', '/')
if x.endswith('/'):
x += '**'
accu = []
for k in x.split('/'):
if k == '**':
accu.append(k)
else:
k = k.replace('.', '[.]').replace('*', '.*').replace('?', '.').replace('+', '\\+')
k = '^%s$' % k
try:
exp = re.compile(k, flags=reflags)
except Exception as e:
raise Errors.WafError('Invalid pattern: %s' % k, e)
else:
accu.append(exp)
ret.append(accu)
return ret
def ant_sub_filter(name, nn):
ret = []
for lst in nn:
if not lst:
pass
elif lst[0] == '**':
ret.append(lst)
if len(lst) > 1:
if lst[1].match(name):
ret.append(lst[2:])
else:
ret.append([])
elif lst[0].match(name):
ret.append(lst[1:])
return ret
def ant_sub_matcher(name, pats):
nacc = ant_sub_filter(name, pats[0])
nrej = ant_sub_filter(name, pats[1])
if [] in nrej:
nacc = []
return [nacc, nrej]
[docs]class Node(object):
"""
This class is organized in two parts:
* The basic methods meant for filesystem access (compute paths, create folders, etc)
* The methods bound to a :py:class:`waflib.Build.BuildContext` (require ``bld.srcnode`` and ``bld.bldnode``)
"""
dict_class = dict
"""
Subclasses can provide a dict class to enable case insensitivity for example.
"""
__slots__ = ('name', 'parent', 'children', 'cache_abspath', 'cache_isdir')
[docs] def __init__(self, name, parent):
"""
.. note:: Use :py:func:`Node.make_node` or :py:func:`Node.find_node` instead of calling this constructor
"""
self.name = name
self.parent = parent
if parent:
if name in parent.children:
raise Errors.WafError('node %s exists in the parent files %r already' % (name, parent))
parent.children[name] = self
[docs] def __setstate__(self, data):
"Deserializes node information, used for persistence"
self.name = data[0]
self.parent = data[1]
if data[2] is not None:
# Issue 1480
self.children = self.dict_class(data[2])
[docs] def __getstate__(self):
"Serializes node information, used for persistence"
return (self.name, self.parent, getattr(self, 'children', None))
[docs] def __str__(self):
"""
String representation (abspath), for debugging purposes
:rtype: string
"""
return self.abspath()
[docs] def __repr__(self):
"""
String representation (abspath), for debugging purposes
:rtype: string
"""
return self.abspath()
[docs] def __copy__(self):
"""
Provided to prevent nodes from being copied
:raises: :py:class:`waflib.Errors.WafError`
"""
raise Errors.WafError('nodes are not supposed to be copied')
[docs] def read(self, flags='r', encoding='latin-1'):
"""
Reads and returns the contents of the file represented by this node, see :py:func:`waflib.Utils.readf`::
def build(bld):
bld.path.find_node('wscript').read()
:param flags: Open mode
:type flags: string
:param encoding: encoding value for Python3
:type encoding: string
:rtype: string or bytes
:return: File contents
"""
return Utils.readf(self.abspath(), flags, encoding)
[docs] def write(self, data, flags='w', encoding='latin-1'):
"""
Writes data to the file represented by this node, see :py:func:`waflib.Utils.writef`::
def build(bld):
bld.path.make_node('foo.txt').write('Hello, world!')
:param data: data to write
:type data: string
:param flags: Write mode
:type flags: string
:param encoding: encoding value for Python3
:type encoding: string
"""
Utils.writef(self.abspath(), data, flags, encoding)
[docs] def read_json(self, convert=True, encoding='utf-8'):
"""
Reads and parses the contents of this node as JSON (Python ≥ 2.6)::
def build(bld):
bld.path.find_node('abc.json').read_json()
Note that this by default automatically decodes unicode strings on Python2, unlike what the Python JSON module does.
:type convert: boolean
:param convert: Prevents decoding of unicode strings on Python2
:type encoding: string
:param encoding: The encoding of the file to read. This default to UTF8 as per the JSON standard
:rtype: object
:return: Parsed file contents
"""
import json # Python 2.6 and up
object_pairs_hook = None
if convert and sys.hexversion < 0x3000000:
try:
_type = unicode
except NameError:
_type = str
def convert(value):
if isinstance(value, list):
return [convert(element) for element in value]
elif isinstance(value, _type):
return str(value)
else:
return value
def object_pairs(pairs):
return dict((str(pair[0]), convert(pair[1])) for pair in pairs)
object_pairs_hook = object_pairs
return json.loads(self.read(encoding=encoding), object_pairs_hook=object_pairs_hook)
[docs] def write_json(self, data, pretty=True):
"""
Writes a python object as JSON to disk (Python ≥ 2.6) as UTF-8 data (JSON standard)::
def build(bld):
bld.path.find_node('xyz.json').write_json(199)
:type data: object
:param data: The data to write to disk
:type pretty: boolean
:param pretty: Determines if the JSON will be nicely space separated
"""
import json # Python 2.6 and up
indent = 2
separators = (',', ': ')
sort_keys = pretty
newline = os.linesep
if not pretty:
indent = None
separators = (',', ':')
newline = ''
output = json.dumps(data, indent=indent, separators=separators, sort_keys=sort_keys) + newline
self.write(output, encoding='utf-8')
[docs] def exists(self):
"""
Returns whether the Node is present on the filesystem
:rtype: bool
"""
return os.path.exists(self.abspath())
[docs] def isdir(self):
"""
Returns whether the Node represents a folder
:rtype: bool
"""
return os.path.isdir(self.abspath())
[docs] def chmod(self, val):
"""
Changes the file/dir permissions::
def build(bld):
bld.path.chmod(493) # 0755
"""
os.chmod(self.abspath(), val)
[docs] def delete(self, evict=True):
"""
Removes the file/folder from the filesystem (equivalent to `rm -rf`), and remove this object from the Node tree.
Do not use this object after calling this method.
"""
try:
try:
if os.path.isdir(self.abspath()):
shutil.rmtree(self.abspath())
else:
os.remove(self.abspath())
except OSError:
if os.path.exists(self.abspath()):
raise
finally:
if evict:
self.evict()
[docs] def evict(self):
"""
Removes this node from the Node tree
"""
del self.parent.children[self.name]
[docs] def suffix(self):
"""
Returns the file rightmost extension, for example `a.b.c.d → .d`
:rtype: string
"""
k = max(0, self.name.rfind('.'))
return self.name[k:]
[docs] def height(self):
"""
Returns the depth in the folder hierarchy from the filesystem root or from all the file drives
:returns: filesystem depth
:rtype: integer
"""
d = self
val = -1
while d:
d = d.parent
val += 1
return val
[docs] def listdir(self):
"""
Lists the folder contents
:returns: list of file/folder names ordered alphabetically
:rtype: list of string
"""
lst = Utils.listdir(self.abspath())
lst.sort()
return lst
[docs] def mkdir(self):
"""
Creates a folder represented by this node. Intermediate folders are created as needed.
:raises: :py:class:`waflib.Errors.WafError` when the folder is missing
"""
if self.isdir():
return
try:
self.parent.mkdir()
except OSError:
pass
if self.name:
try:
os.makedirs(self.abspath())
except OSError:
pass
if not self.isdir():
raise Errors.WafError('Could not create the directory %r' % self)
try:
self.children
except AttributeError:
self.children = self.dict_class()
[docs] def find_node(self, lst):
"""
Finds a node on the file system (files or folders), and creates the corresponding Node objects if it exists
:param lst: relative path
:type lst: string or list of string
:returns: The corresponding Node object or None if no entry was found on the filesystem
:rtype: :py:class:´waflib.Node.Node´
"""
if isinstance(lst, str):
lst = [x for x in Utils.split_path(lst) if x and x != '.']
if lst and lst[0].startswith('\\\\') and not self.parent:
node = self.ctx.root.make_node(lst[0])
node.cache_isdir = True
return node.find_node(lst[1:])
cur = self
for x in lst:
if x == '..':
cur = cur.parent or cur
continue
try:
ch = cur.children
except AttributeError:
cur.children = self.dict_class()
else:
try:
cur = ch[x]
continue
except KeyError:
pass
# optimistic: create the node first then look if it was correct to do so
cur = self.__class__(x, cur)
if not cur.exists():
cur.evict()
return None
if not cur.exists():
cur.evict()
return None
return cur
[docs] def make_node(self, lst):
"""
Returns or creates a Node object corresponding to the input path without considering the filesystem.
:param lst: relative path
:type lst: string or list of string
:rtype: :py:class:´waflib.Node.Node´
"""
if isinstance(lst, str):
lst = [x for x in Utils.split_path(lst) if x and x != '.']
cur = self
for x in lst:
if x == '..':
cur = cur.parent or cur
continue
try:
cur = cur.children[x]
except AttributeError:
cur.children = self.dict_class()
except KeyError:
pass
else:
continue
cur = self.__class__(x, cur)
return cur
[docs] def search_node(self, lst):
"""
Returns a Node previously defined in the data structure. The filesystem is not considered.
:param lst: relative path
:type lst: string or list of string
:rtype: :py:class:´waflib.Node.Node´ or None if there is no entry in the Node datastructure
"""
if isinstance(lst, str):
lst = [x for x in Utils.split_path(lst) if x and x != '.']
cur = self
for x in lst:
if x == '..':
cur = cur.parent or cur
else:
try:
cur = cur.children[x]
except (AttributeError, KeyError):
return None
return cur
[docs] def path_from(self, node):
"""
Path of this node seen from the other::
def build(bld):
n1 = bld.path.find_node('foo/bar/xyz.txt')
n2 = bld.path.find_node('foo/stuff/')
n1.path_from(n2) # '../bar/xyz.txt'
:param node: path to use as a reference
:type node: :py:class:`waflib.Node.Node`
:returns: a relative path or an absolute one if that is better
:rtype: string
"""
c1 = self
c2 = node
c1h = c1.height()
c2h = c2.height()
lst = []
up = 0
while c1h > c2h:
lst.append(c1.name)
c1 = c1.parent
c1h -= 1
while c2h > c1h:
up += 1
c2 = c2.parent
c2h -= 1
while not c1 is c2:
lst.append(c1.name)
up += 1
c1 = c1.parent
c2 = c2.parent
if c1.parent:
lst.extend(['..'] * up)
lst.reverse()
return os.sep.join(lst) or '.'
else:
return self.abspath()
def abspath(self):
"""
Returns the absolute path. A cache is kept in the context as ``cache_node_abspath``
:rtype: string
"""
try:
return self.cache_abspath
except AttributeError:
pass
# think twice before touching this (performance + complexity + correctness)
if not self.parent:
val = os.sep
elif not self.parent.name:
val = os.sep + self.name
else:
val = self.parent.abspath() + os.sep + self.name
self.cache_abspath = val
return val
if Utils.is_win32:
[docs] def abspath(self):
try:
return self.cache_abspath
except AttributeError:
pass
if not self.parent:
val = ''
elif not self.parent.name:
val = self.name + os.sep
else:
val = self.parent.abspath().rstrip(os.sep) + os.sep + self.name
self.cache_abspath = val
return val
[docs] def is_child_of(self, node):
"""
Returns whether the object belongs to a subtree of the input node::
def build(bld):
node = bld.path.find_node('wscript')
node.is_child_of(bld.path) # True
:param node: path to use as a reference
:type node: :py:class:`waflib.Node.Node`
:rtype: bool
"""
p = self
diff = self.height() - node.height()
while diff > 0:
diff -= 1
p = p.parent
return p is node
[docs] def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remove=True, quiet=False):
"""
Recursive method used by :py:meth:`waflib.Node.ant_glob`.
:param accept: function used for accepting/rejecting a node, returns the patterns that can be still accepted in recursion
:type accept: function
:param maxdepth: maximum depth in the filesystem (25)
:type maxdepth: int
:param pats: list of patterns to accept and list of patterns to exclude
:type pats: tuple
:param dir: return folders too (False by default)
:type dir: bool
:param src: return files (True by default)
:type src: bool
:param remove: remove files/folders that do not exist (True by default)
:type remove: bool
:param quiet: disable build directory traversal warnings (verbose mode)
:type quiet: bool
:returns: A generator object to iterate from
:rtype: iterator
"""
dircont = self.listdir()
try:
lst = set(self.children.keys())
except AttributeError:
self.children = self.dict_class()
else:
if remove:
for x in lst - set(dircont):
self.children[x].evict()
for name in dircont:
npats = accept(name, pats)
if npats and npats[0]:
accepted = [] in npats[0]
node = self.make_node([name])
isdir = node.isdir()
if accepted:
if isdir:
if dir:
yield node
elif src:
yield node
if isdir:
node.cache_isdir = True
if maxdepth:
for k in node.ant_iter(accept=accept, maxdepth=maxdepth - 1, pats=npats, dir=dir, src=src, remove=remove, quiet=quiet):
yield k
[docs] def ant_glob(self, *k, **kw):
"""
Finds files across folders and returns Node objects:
* ``**/*`` find all files recursively
* ``**/*.class`` find all files ending by .class
* ``..`` find files having two dot characters
For example::
def configure(cfg):
# find all .cpp files
cfg.path.ant_glob('**/*.cpp')
# find particular files from the root filesystem (can be slow)
cfg.root.ant_glob('etc/*.txt')
# simple exclusion rule example
cfg.path.ant_glob('*.c*', excl=['*.c'], src=True, dir=False)
For more information about the patterns, consult http://ant.apache.org/manual/dirtasks.html
Please remember that the '..' sequence does not represent the parent directory::
def configure(cfg):
cfg.path.ant_glob('../*.h') # incorrect
cfg.path.parent.ant_glob('*.h') # correct
The Node structure is itself a filesystem cache, so certain precautions must
be taken while matching files in the build or installation phases.
Nodes objects that do have a corresponding file or folder are garbage-collected by default.
This garbage collection is usually required to prevent returning files that do not
exist anymore. Yet, this may also remove Node objects of files that are yet-to-be built.
This typically happens when trying to match files in the build directory,
but there are also cases when files are created in the source directory.
Run ``waf -v`` to display any warnings, and try consider passing ``remove=False``
when matching files in the build directory.
Since ant_glob can traverse both source and build folders, it is a best practice
to call this method only from the most specific build node::
def build(bld):
# traverses the build directory, may need ``remove=False``:
bld.path.ant_glob('project/dir/**/*.h')
# better, no accidental build directory traversal:
bld.path.find_node('project/dir').ant_glob('**/*.h') # best
In addition, files and folders are listed immediately. When matching files in the
build folders, consider passing ``generator=True`` so that the generator object
returned can defer computation to a later stage. For example::
def build(bld):
bld(rule='tar xvf ${SRC}', source='arch.tar')
bld.add_group()
gen = bld.bldnode.ant_glob("*.h", generator=True, remove=True)
# files will be listed only after the arch.tar is unpacked
bld(rule='ls ${SRC}', source=gen, name='XYZ')
:param incl: ant patterns or list of patterns to include
:type incl: string or list of strings
:param excl: ant patterns or list of patterns to exclude
:type excl: string or list of strings
:param dir: return folders too (False by default)
:type dir: bool
:param src: return files (True by default)
:type src: bool
:param maxdepth: maximum depth of recursion
:type maxdepth: int
:param ignorecase: ignore case while matching (False by default)
:type ignorecase: bool
:param generator: Whether to evaluate the Nodes lazily
:type generator: bool
:param remove: remove files/folders that do not exist (True by default)
:type remove: bool
:param quiet: disable build directory traversal warnings (verbose mode)
:type quiet: bool
:returns: The corresponding Node objects as a list or as a generator object (generator=True)
:rtype: by default, list of :py:class:`waflib.Node.Node` instances
"""
src = kw.get('src', True)
dir = kw.get('dir')
excl = kw.get('excl', exclude_regs)
incl = k and k[0] or kw.get('incl', '**')
remove = kw.get('remove', True)
maxdepth = kw.get('maxdepth', 25)
ignorecase = kw.get('ignorecase', False)
quiet = kw.get('quiet', False)
pats = (ant_matcher(incl, ignorecase), ant_matcher(excl, ignorecase))
if kw.get('generator'):
return Utils.lazy_generator(self.ant_iter, (ant_sub_matcher, maxdepth, pats, dir, src, remove, quiet))
it = self.ant_iter(ant_sub_matcher, maxdepth, pats, dir, src, remove, quiet)
if kw.get('flat'):
# returns relative paths as a space-delimited string
# prefer Node objects whenever possible
return ' '.join(x.path_from(self) for x in it)
return list(it)
# ----------------------------------------------------------------------------
# the methods below require the source/build folders (bld.srcnode/bld.bldnode)
[docs] def is_src(self):
"""
Returns True if the node is below the source directory. Note that ``!is_src() ≠ is_bld()``
:rtype: bool
"""
cur = self
x = self.ctx.srcnode
y = self.ctx.bldnode
while cur.parent:
if cur is y:
return False
if cur is x:
return True
cur = cur.parent
return False
[docs] def is_bld(self):
"""
Returns True if the node is below the build directory. Note that ``!is_bld() ≠ is_src()``
:rtype: bool
"""
cur = self
y = self.ctx.bldnode
while cur.parent:
if cur is y:
return True
cur = cur.parent
return False
[docs] def get_src(self):
"""
Returns the corresponding Node object in the source directory (or self if already
under the source directory). Use this method only if the purpose is to create
a Node object (this is common with folders but not with files, see ticket 1937)
:rtype: :py:class:`waflib.Node.Node`
"""
cur = self
x = self.ctx.srcnode
y = self.ctx.bldnode
lst = []
while cur.parent:
if cur is y:
lst.reverse()
return x.make_node(lst)
if cur is x:
return self
lst.append(cur.name)
cur = cur.parent
return self
[docs] def get_bld(self):
"""
Return the corresponding Node object in the build directory (or self if already
under the build directory). Use this method only if the purpose is to create
a Node object (this is common with folders but not with files, see ticket 1937)
:rtype: :py:class:`waflib.Node.Node`
"""
cur = self
x = self.ctx.srcnode
y = self.ctx.bldnode
lst = []
while cur.parent:
if cur is y:
return self
if cur is x:
lst.reverse()
return self.ctx.bldnode.make_node(lst)
lst.append(cur.name)
cur = cur.parent
# the file is external to the current project, make a fake root in the current build directory
lst.reverse()
if lst and Utils.is_win32 and len(lst[0]) == 2 and lst[0].endswith(':'):
lst[0] = lst[0][0]
return self.ctx.bldnode.make_node(['__root__'] + lst)
[docs] def find_resource(self, lst):
"""
Use this method in the build phase to find source files corresponding to the relative path given.
First it looks up the Node data structure to find any declared Node object in the build directory.
If None is found, it then considers the filesystem in the source directory.
:param lst: relative path
:type lst: string or list of string
:returns: the corresponding Node object or None
:rtype: :py:class:`waflib.Node.Node`
"""
if isinstance(lst, str):
lst = [x for x in Utils.split_path(lst) if x and x != '.']
node = self.get_bld().search_node(lst)
if not node:
node = self.get_src().find_node(lst)
if node and node.isdir():
return None
return node
[docs] def find_or_declare(self, lst):
"""
Use this method in the build phase to declare output files which
are meant to be written in the build directory.
This method creates the Node object and its parent folder
as needed.
:param lst: relative path
:type lst: string or list of string
"""
if isinstance(lst, str) and os.path.isabs(lst):
node = self.ctx.root.make_node(lst)
else:
node = self.get_bld().make_node(lst)
node.parent.mkdir()
return node
[docs] def find_dir(self, lst):
"""
Searches for a folder on the filesystem (see :py:meth:`waflib.Node.Node.find_node`)
:param lst: relative path
:type lst: string or list of string
:returns: The corresponding Node object or None if there is no such folder
:rtype: :py:class:`waflib.Node.Node`
"""
if isinstance(lst, str):
lst = [x for x in Utils.split_path(lst) if x and x != '.']
node = self.find_node(lst)
if node and not node.isdir():
return None
return node
# helpers for building things
[docs] def change_ext(self, ext, ext_in=None):
"""
Declares a build node with a distinct extension; this is uses :py:meth:`waflib.Node.Node.find_or_declare`
:return: A build node of the same path, but with a different extension
:rtype: :py:class:`waflib.Node.Node`
"""
name = self.name
if ext_in is None:
k = name.rfind('.')
if k >= 0:
name = name[:k] + ext
else:
name = name + ext
else:
name = name[:- len(ext_in)] + ext
return self.parent.find_or_declare([name])
[docs] def bldpath(self):
"""
Returns the relative path seen from the build directory ``src/foo.cpp``
:rtype: string
"""
return self.path_from(self.ctx.bldnode)
[docs] def srcpath(self):
"""
Returns the relative path seen from the source directory ``../src/foo.cpp``
:rtype: string
"""
return self.path_from(self.ctx.srcnode)
[docs] def relpath(self):
"""
If a file in the build directory, returns :py:meth:`waflib.Node.Node.bldpath`,
else returns :py:meth:`waflib.Node.Node.srcpath`
:rtype: string
"""
cur = self
x = self.ctx.bldnode
while cur.parent:
if cur is x:
return self.bldpath()
cur = cur.parent
return self.srcpath()
[docs] def bld_dir(self):
"""
Equivalent to self.parent.bldpath()
:rtype: string
"""
return self.parent.bldpath()
def h_file(self):
"""
See :py:func:`waflib.Utils.h_file`
:return: a hash representing the file contents
:rtype: string or bytes
"""
return Utils.h_file(self.abspath())
[docs] def get_bld_sig(self):
"""
Returns a signature (see :py:meth:`waflib.Node.Node.h_file`) for the purpose
of build dependency calculation. This method uses a per-context cache.
:return: a hash representing the object contents
:rtype: string or bytes
"""
# previous behaviour can be set by returning self.ctx.node_sigs[self] when a build node
try:
cache = self.ctx.cache_sig
except AttributeError:
cache = self.ctx.cache_sig = {}
try:
ret = cache[self]
except KeyError:
p = self.abspath()
try:
ret = cache[self] = self.h_file()
except EnvironmentError:
if self.isdir():
# allow folders as build nodes, do not use the creation time
st = os.stat(p)
ret = cache[self] = Utils.h_list([p, st.st_ino, st.st_mode])
return ret
raise
return ret
pickle_lock = Utils.threading.Lock()
"""Lock mandatory for thread-safe node serialization"""
class Nod3(Node):
"""Mandatory subclass for thread-safe node serialization"""
pass # do not remove
