+

+ +

Source code for diffpy.pdfgetx.functs

+#!/usr/bin/env python
+##############################################################################
+#
+# diffpy.pdfgetx    by DANSE Diffraction group
+#                   Simon J. L. Billinge
+#                   (c) 2008 Trustees of the Columbia University
+#                   in the City of New York.  All rights reserved.
+#
+# File coded by:    Timur Dykhne
+#
+# See AUTHORS.rst for a list of people who contributed.
+# See LICENSENOTICE.rst for license information.
+#
+##############################################################################
+
+import os.path
+import re
+import sys
+
+
+def composition_analysis(compstring):
+    """Pulls out elements and their ratios from the config file.
+
+    compstring   -- chemical composition of the sample, e.g.,
+                    "NaCl", "H2SO4", "(LaCa)1/2 Mn O3".  Blank
+                    characters are ignored, unit counts can be omitted.
+                    It is critical to use proper upper-lower case for atom
+                    symbols as this is used to delimit them in the formula.
+
+    Returns a list of atom symbols and a corresponding list of their counts.
+    """
+    # check if parentheses are balanced
+    if compstring.count("(") != compstring.count(")"):
+        emsgbal = 'imbalanced parentheses in "%s"' % compstring
+        raise ValueError(emsgbal)
+    # make sure there is at least one uppercase character in the compstring
+    upcasechars = any(c.isupper() for c in compstring)
+    nonblank = any(not c.isspace() for c in compstring)
+    if not upcasechars and nonblank:
+        emsg = 'invalid chemical composition "%s"' % compstring
+        raise ValueError(emsg)
+    if "(" in compstring:
+        head, body, count, tail = _parse_parentheses(compstring)
+        ca_head = composition_analysis(head)
+        ca_body = composition_analysis(body)
+        ca_tail = composition_analysis(tail)
+        names = ca_head[0] + ca_body[0] + ca_tail[0]
+        fractions = ca_head[1] + [count * x for x in ca_body[1]] + ca_tail[1]
+        return names, fractions
+    # simple composition string here
+    assert ")" not in compstring
+    # split at every upper-case letter, possibly followed by a lower case
+    # one and charge specification
+    namefracs = re.split(_re_atomtype, compstring)[1:]
+    names = namefracs[0::2]
+    # use unit count when empty, convert to float otherwise
+    fractions = [_parse_fraction(w) for w in namefracs[1::2]]
+    return names, fractions
+
+
+_re_atomtype = "([A-Z][a-z]?(?:[1-8]?[+-])?)"
+
+
+def _parse_parentheses(compstring):
+    """Split fragment of chemical formula at the first pair of
+    parentheses.
+
+    compstring   -- chemical formula that contains one or more pairs
+                    of parentheses.
+
+    Return a tuple of (head, body, count, tail) where
+
+    head     -- is the input string up to the opening parenthesis
+    body     -- compstring fragment enclosed by the first parentheses.
+                It may contain more parentheses.
+    count    -- floating point ratio following the closing parenthesis.
+                Defaults to 1 when not present.
+    tail     -- trailing part of the compstring after the count.
+    """
+    psign = {"(": +1, ")": -1}
+    ptotal = 0
+    p1 = compstring.index("(")
+    for p2, c in enumerate(compstring[p1:], p1):
+        ptotal += psign.get(c, 0)
+        if ptotal == 0:
+            break
+    mx = re.search(_re_atomtype + r"|\(", compstring[p2:])
+    p3 = (mx.start() + p2) if mx else len(compstring)
+    head = compstring[:p1]
+    body = compstring[p1 + 1 : p2]
+    scnt = compstring[p2 + 1 : p3]
+    count = _parse_fraction(scnt)
+    tail = compstring[p3:]
+    rv = head, body, count, tail
+    return rv
+
+
+def _parse_fraction(s):
+    """Convert fraction strings in chemical formulas to floats.
+
+    s    -- string convertible to float or a fraction "1/3".
+
+    Return floating point value parsed from s.  Return 1.0 when
+    s is empty or of space characters only.
+    """
+    from fractions import Fraction
+
+    if "/" in s:
+        rv = float(Fraction(s))
+    elif s.strip():
+        rv = float(s)
+    else:
+        rv = 1.0
+    return rv
+
+
+
+[docs]
+def findfiles(patterns=(), path=".", dotfiles=False):
+    """Find files that match all specified patterns.
+
+    Pattern syntax:
+
+    * ``^start`` - match "start" only at the beginning of the string.
+    * ``end$`` - match "end" only at the end of string.
+    * ``<7>`` - match number 7 preceded by any number of leading zeros.
+    * ``<1-34>`` - match an integer range from 1 to 34 inclusive.
+    * ``<7->`` - match an integer greater or equal 7.
+    * ``<->`` - match any integer.
+    * ``+`` - start a new group of patterns to match more files.
+    * ``dir/`` - set search path effective from the current pattern group.
+
+    All integer ranges ``<N-M>`` above allow one or more leading zeros.
+    The range syntax does not support matching of negative numbers.
+
+    Parameters
+    ----------
+    patterns : iterable of strings or str, optional
+        String patterns that must all match in returned filenames.
+        Can be also a single string with patterns separated by
+        whitespace characters.  When empty match all files in the
+        current directory or in the `path`.  A single ``+`` starts
+        a new pattern group for additional matches.  Each pattern
+        group may have one entry containing ``/``, for example,
+        ``dir/`` or ``./``, which sets the search directory for
+        this and subsequent pattern groups.  When pattern group
+        contains only the path entry it reuses file patterns from
+        the previous group.
+    path : str, optional
+        Directory to be searched for the files.  The default is "."
+        to search the current directory.
+    dotfiles : bool, optional
+        When True search also the hidden "." starting files.  These
+        files are by default ignored, but can be explicitly selected
+        by adding the ``^.`` pattern.
+
+    Returns
+    -------
+    filenames : list
+        The list of matching filenames.  Return all files when
+        `patterns` are not specified.
+    """
+    import shlex
+    from itertools import groupby
+    from pathlib import PurePath
+
+    from diffpy.pdfgetx.multipattern import MultiPattern
+
+    if isinstance(patterns, str):
+        p1 = shlex.split(patterns)
+    else:
+        p1 = list(patterns)
+    dirfiles = {}
+    # split patterns into groups separated by the '+' item
+    pgroups = [list(g) for k, g in groupby(p1, key=lambda x: x != "+") if k]
+    pgroups = pgroups or [[]]
+    cpath = asunicodepath(path)
+    cpatterns = []
+    files = []
+
+    # Support backslash to identify directory entry on Windows
+    def hasslash(s):
+        return PurePath(s).name != s
+
+    for pg in pgroups:
+        gpath = tuple(p for p in pg if hasslash(p))
+        if len(gpath) > 1:
+            emsg = "Too many directories {!r}.".format(gpath)
+            raise ValueError(emsg)
+        gpatterns = [p for p in pg if p not in gpath]
+        # update current path and patterns when provided
+        if gpath:
+            (cpath,) = gpath
+        if gpatterns:
+            cpatterns = gpatterns
+        ap = os.path.abspath(cpath)
+        if ap not in dirfiles:
+            dirfiles[ap] = os.listdir(ap)
+            dirfiles[ap].sort(key=sortKeyNumericString)
+        keepdotfiles = dotfiles or any(p.startswith("^.") for p in cpatterns)
+        mp = MultiPattern(cpatterns)
+
+        def fileok(f):
+            path = PurePath(f)
+            return (not path.name.startswith(".") or keepdotfiles) and mp.match(str(path))
+
+        files += (os.path.normpath(os.path.join(cpath, f)) for f in dirfiles[ap] if fileok(f))
+    # make matching files unique and filter out any directories
+    rv = [f for f in unique_everseen(files) if os.path.isfile(f)]
+    return rv
+
+
+
+
+[docs]
+def loaddata(filename, minrows=10, usecols=None, **kwargs):
+    """Find and load data from a text file.
+
+    The data reading starts at the first matrix block of at least minrows rows
+    and constant number of columns.  This seems to work for most of the
+    datafiles including those generated by PDFGetX2.
+
+    Parameters
+    ----------
+
+    filename : str
+        Name of the file to load the text data from.
+    minrows : int, optional
+        Minimum number of rows in the first data block, by default 10.
+        All rows must have the same number of floating point values.
+    usecols : int, str, slice, iterable, optional
+        Indices or names of the columns to be loaded from the data block,
+        the default is all columns.  Data blocks that do not contain
+        sufficient number of columns are skipped.  When usecols contain
+        string items, they are translated to column indices by looking
+        up a header line preceding the data block.  String items formatted
+        as ``i:j:k`` are converted to `slice` objects.  When usecols type
+        is string it is split to a list of names at comma and whitespace
+        characters.
+    unpack : bool, optional
+        Return data as a sequence of columns that allows tuple unpacking
+        such as ``x, y = loaddata(FILENAME, unpack=True)``.  Note that
+        transposing the loaded array as ``loaddata(FILENAME).T`` has
+        the same effect.  The default is False.
+    kwargs : misc, optional
+        Extra keyword arguments that are passed to `numpy.loadtxt`.
+
+    Returns
+    -------
+    data : numpy.ndarray
+        The data block loaded from the text file.
+
+    See also
+    --------
+    numpy.loadtxt
+    """
+    from numpy import array, loadtxt
+
+    # determine the arguments
+    delimiter = kwargs.get("delimiter")
+    if delimiter is not None:
+        delimiter = delimiter.encode("utf-8")
+    # required at least one column of floating point values
+    mincv = (1, 1)
+    # but if usecols is specified, require sufficient number of columns
+    # where the used columns contain floats
+    if usecols is not None:
+        usecols = _resolveUsedColumns(usecols, filename)
+        ucx = _expandSliceItems(usecols) or [0]
+        hiidx = max(-min(ucx), max(ucx) + 1)
+        mincv = (hiidx, len(set(ucx)))
+    _ucx_cache = {}
+
+    def getucx(n):
+        return _ucx_cache.setdefault(n, _expandSliceItems(usecols, n))
+
+    # Check if a line consists of floats only and return their count
+    # Return zero if some strings cannot be converted.
+    def countcolumnsvalues(line):
+        words = line.split(delimiter)
+        # remove trailing blank columns
+        while words and not words[-1].strip():
+            words.pop(-1)
+        nc = len(words)
+        # build a list of selected words `wsel`
+        wsel = words
+        if usecols is not None:
+            try:
+                wsel = [words[i] for i in getucx(nc)]
+            except IndexError:
+                return 0, 0
+        # test if all selected words can be converted to float
+        try:
+            nv = len([float(w) for w in wsel])
+        except ValueError:
+            nc = nv = 0
+        return nc, nv
+
+    # make sure fid gets cleaned up
+    with open(filename, "rb") as fid:
+        # search for the start of datablock
+        start = ncvblock = None
+        fpos = nrows = 0
+        for line in fid:
+            fpos += len(line)
+            ncv = countcolumnsvalues(line)
+            if ncv < mincv:
+                start = None
+                continue
+            # ncv is acceptable here, require the same number of columns
+            # throughout the datablock
+            if start is None or ncv != ncvblock:
+                ncvblock = ncv
+                nrows = 0
+                start = fpos - len(line)
+            nrows += 1
+            # block was found here!
+            if nrows >= minrows:
+                break
+        # Return an empty array when no data found.
+        # loadtxt would otherwise raise an exception on loading from EOF.
+        if start is None:
+            rv = array([], dtype=float)
+        else:
+            fid.seek(start)
+            # always use usecols argument so that loadtxt does not crash
+            # in case of trailing delimiters.
+            n = ncvblock[0]
+            kwargs["usecols"] = list(range(n)) if usecols is None else getucx(n)
+            rv = loadtxt(fid, **kwargs)
+    return rv
+
+
+
+def findColumnNames(headlines):
+    """Obtain a list of unique column names from a list of header
+    lines."""
+    mxtwospace = re.compile(r"(?:\t|  )\s*")
+    colnames = []
+    for line in reversed(headlines):
+        words = line.split()
+        if sum(isfloat(word) for word in words) >= len(words) // 2:
+            colnames = []
+            continue
+        if line.lstrip("#").split()[:2] == ["start", "data"]:
+            continue
+        if len(words) > 1 and words[0] == "#L":
+            w1 = words[1:]
+            s1 = line.replace("#L", "", 1).strip()
+            w2 = mxtwospace.split(s1)
+            colnames = w2 if len(w2) > 1 else w1
+            break
+        if words and words[0].startswith("#"):
+            words[0] = words[0].lstrip("#").strip()
+            if not words[0]:
+                words.pop(0)
+        if words and all([w[:1].isalpha() for w in words]):
+            colnames = words
+            break
+    return colnames
+
+
+def loadColumnNames(filename):
+    """Obtain unique column names from a text file header section."""
+    with open(filename) as fp:
+        headlines = fp.readlines(32768)
+    rv = findColumnNames(headlines)
+    return rv
+
+
+def _resolveUsedColumns(usecols, filename=None):
+    """Translate any column names to corresponding integer indices.
+
+    usecols  -- an iterable of integers and/or strings.  Can be also
+                an integer or a comma or space-separated string.
+                String names are translated to integers according
+                to column names find in the filename.
+    filename -- text data file where column names are looked up.
+                Do not translate column names when not specified.
+
+    Return a new usecols list containing integers only.
+    Raise ValueError if a column name cannot be resolved.
+    """
+
+    def isstr(s):
+        return isinstance(s, str)
+
+    # (1) convert usecols to a list
+    uc1 = [usecols] if isstr(usecols) else list(usecols) if isiterable(usecols) else [usecols]
+    # (2) split string items at ","
+    uc2 = sum((c.replace(",", " ").split() if isstr(c) else [c] for c in uc1), [])
+    # (3) convert integer string items to int
+    uc3 = [int(c) if isstr(c) and isint(c) else c for c in uc2]
+    # (4) convert slice-like string items to slices
+    uc4 = []
+    for c in uc3:
+        if not isstr(c):
+            uc4.append(c)
+            continue
+        if c.count(":") == 0 or c.count(":") > 2:
+            uc4.append(c)
+            continue
+        a0 = [w.strip() for w in c.split(":")]
+        if not all(w == "" or isint(w) for w in a0):
+            uc4.append(c)
+            continue
+        # pad a0 to have 3 elements for sure
+        a0 += 3 * [""]
+        a1 = [int(w) if w else None for w in a0[:3]]
+        ss = slice(*a1)
+        uc4.append(ss)
+    # if there are no string identifiers we are done here
+    if filename is None or all(not isstr(c) for c in uc4):
+        return uc4
+    # (5) otherwise we need to translate column names to indices
+    colnames = loadColumnNames(filename)
+    colnamesnum = [(n, i) for i, n in enumerate(colnames)]
+    colnamesfixed = _fixColumnNames(colnames)
+    # use the first matching column in case of repeated names.
+    cnmtoidx = dict(reversed(colnamesnum))
+    # fixed names do not repeat so use them as they are.
+    cnmtoidx.update((n, i) for i, n in enumerate(colnamesfixed))
+    uc5 = [cnmtoidx.get(c, c) if isstr(c) else c for c in uc4]
+    # any string items left are bad names
+    badnames = [c for c in uc5 if isstr(c)]
+    if badnames:
+        emsg = "invalid column names: %s" % ", ".join(badnames)
+        raise ValueError(emsg)
+    # phew, done here.
+    return uc5
+
+
+def _fixColumnNames(names):
+    """Remove any parentheses and non-alphabetic symbols from a list of
+    names. Ensure the new list of names is still unique.
+
+    Return a new list of names.
+    """
+    rv = []
+    nameindices = {}
+    mxpar = re.compile(r"[(][^)]*[)]")
+    mxnoword = re.compile(r"\W")
+    for i, nm in enumerate(names):
+        while mxpar.search(nm):
+            nm = mxpar.sub("", nm)
+        nm = mxnoword.sub("", nm)
+        rv.append(nm)
+        nameindices.setdefault(nm, []).append(i)
+    for nm, indices in nameindices.items():
+        for i, idx in enumerate(indices[1:]):
+            rv[idx] += str(i + 1)
+    return rv
+
+
+def _expandSliceItems(lst, size=None):
+    """Replace slice items in a list with corresponding indices.
+
+    When `size` is not specified, extract available *start* and
+    *stop* values to determine the necessary length of the sliced
+    array.
+
+    Parameters
+    ----------
+    lst : list
+        The list which may contain `slice` objects.
+    size : int, optional
+        The size of the iterable to be sliced.  When specified
+        expand slice objects to corresponding indices.
+
+    Returns
+    -------
+    lst2 : list
+        The new list with slice items expanded.  All other items
+        in the input list `lst` are passed unmodified.
+    """
+    rv = []
+    indices = list(range(size)) if size is not None else []
+    for x in lst:
+        if not isinstance(x, slice):
+            rv.append(x)
+            continue
+        # x is a slice here
+        # (1) expand indices when size is specified
+        if size is not None:
+            rv += indices[x]
+            continue
+        # (2) otherwise extract known slice bounds
+        if x.start is not None:
+            rv.append(x.start)
+        # keep it simple and process only positive x.stop
+        if x.stop is not None and x.stop > 0:
+            if x.step is None or x.step > 0:
+                n = x.stop - 1
+                rv.append(n)
+    return rv
+
+
+def twoThetaToQ(twotheta, wavelength):
+    """Converts an array from having values in Two-Theta to Q in
+    angstroms.
+
+    twotheta   -- An array of two-theta values in degrees
+    wavelength -- The wavelength of the x-ray beam that was used in Angstroms.
+
+    Returns an array of Q values in 1/A units.
+    """
+    from numpy import pi, sin
+
+    Q = 4 * pi * sin(twotheta / 2.0 * pi / 180) / wavelength
+    return Q
+
+
+def pylab_draw_show():
+    """Same as calling pyplot draw and show functions."""
+    import matplotlib.pyplot as plt
+    from bg_mpl_stylesheets.styles import all_styles
+
+    plt.style.use(all_styles["bg-style"])
+    plt.rcParams["lines.linewidth"] = 1.5
+
+    plt.draw()
+    plt.show()
+    return
+
+
+def unique_everseen(iterable, key=None):
+    """Return a generator to unique ordered elements in an iterable.
+
+    iterable -- an iterable object
+    key      -- function that returns key for finding unique items.
+                Use the item value when None.
+
+    unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    unique_everseen('ABBCcAD', str.lower) --> A B C D
+    """
+    seen = set()
+    for element in iterable:
+        k = element if key is None else key(element)
+        if k not in seen:
+            seen.add(k)
+            yield element
+    pass
+
+
+def isiterable(obj):
+    """Return True if object supports iteration protocol."""
+    try:
+        iter(obj)
+        rv = True
+    except TypeError:
+        rv = False
+    return rv
+
+
+def isfloat(x):
+    """Return True if string is convertible to float."""
+    try:
+        float(x)
+        rv = True
+    except ValueError:
+        rv = False
+    return rv
+
+
+def isint(x):
+    """Return True if string is convertible to int."""
+    try:
+        int(x)
+        rv = True
+    except ValueError:
+        rv = False
+    return rv
+
+
+def getbooleanorpass(value, ignore=None):
+    """Convert value to a boolean or pass it when ignored.
+
+    Parameters
+    ----------
+    value : str, number, object
+        The value to be converted to `bool` or passed unchanged.
+    ignore : container or callable, optional
+        Specify values that should be passed unchanged.  When callable
+        pass the `value` as is when `ignore(value)` is true; otherwise
+        check ``value in ignore``.
+
+    Returns
+    -------
+    result : bool or `value`
+
+    Raises
+    ------
+    ValueError
+        When `value` cannot be converted to `bool` and is not ignored.
+    """
+    # check if the value is ignored
+    if ignore is not None:
+        yesignore = ignore(value) if callable(ignore) else (value in ignore)
+        if yesignore:
+            return value
+    # here we need to convert to bool
+    if isint(value):
+        return bool(value)
+    boolean_states = {
+        "1": True,
+        "yes": True,
+        "true": True,
+        "on": True,
+        "0": False,
+        "no": False,
+        "false": False,
+        "off": False,
+    }
+    v = str(value).lower()
+    if v not in boolean_states:
+        emsg = "not a boolean: %s" % value
+        raise ValueError(emsg)
+    return boolean_states[v]
+
+
+def sortKeyNumericString(s):
+    """This function can be used as a key argument for sort to order
+    string items in numeric, rather than alphabetic order.
+
+    s    -- string entry
+
+    Return a key for comparison in sorting.  The string s is split at
+    integer segments that are then converted to integers.  Signs, decimal
+    points and exponents in s are ignored.
+    """
+    if not hasattr(sortKeyNumericString, "rxdigits"):
+        sortKeyNumericString.rxdigits = re.compile(r"(\d+)")
+    rv = sortKeyNumericString.rxdigits.split(s)
+    rv[1::2] = map(int, rv[1::2])
+    return rv
+
+
+def asunicodepath(f):
+    """Convert path to unicode if it has any non-ASCII characters.
+
+    f    -- file or directory path.  May be already unicode or a bytes
+            array.
+
+    Return f if it is already a unicode.  Otherwise decode it to unicode
+    using filesystem encoding.
+    """
+    rv = f
+    if not isinstance(f, str):
+        rv = f.decode(sys.getfilesystemencoding())
+    return rv
+
+
+def multi_delim_split(delims, string):
+    """Extend Python string splitting to splitting on multiple
+    delimiters.
+
+    delims -- an iterable of strings to be used as delimiters for splitting the
+              given string.
+    string -- the string to be split
+
+    Returns a list of strings identical to if str.split(delim) had been called
+    if specifying multiple delimiters in the arguments were supported.
+    """
+    return re.split(r"|".join(re.escape(delim) for delim in delims), string)
+

+ +

+

^	match at the beginning of the string, i.e., `^start` +matches only filenames that start with “start”.
$	match at the end of string, for example, `.chi$` selects +file names ending with “.chi”. A `$` on its own +matches every string and can be used to select all files.
<N>	match number N preceded by any number of leading zeros, +e.g., `<7>` would match in “f7.chi”, “f007.chi”, but not +in “f77.chi”.
<N-M>	match an integer range from N to M inclusive. +The matched number may have one or more leading zeros.
<7->	match number 7 or larger.
<-7>	match number 7 or smaller.
<->	match any integer number.
+	start a new pattern group, for example, `.chi$ + .dat$`
/	set search path. An argument containing the `/` symbol +is taken as the search path, for example, `data/` or +`./`. Each pattern group may provide its own search +path effective for that and any subsequent pattern group.

token	example	definition
@h	dir1/dir2	the input file directory or “.”
@r	dir1/dir2/filename	the input path with extension removed
@e	dat	the input file extension without “.”
@t	filename.dat	the tail component of the input file
@b	filename	the tail component with extension removed
@o	gr	the output extension iq, sq, fq or gr

Source code for diffpy.pdfgetx.functs

Source code for diffpy.pdfgetx.plotdata

All modules for which code is available

Files used in PDF extraction

Configuration file

Input files

Output files

Index

Symbols

A

B

C

D

E

F

G

I

L

M

O

P

Q

R

S

T

V

W

diffpy.pdfgetx

Table of contents

Indices

Installation

Software requirements

Installation

IPython magic command

Interactive mode

Introduction

License notice

Authors

References

Options and parameters

Program operation

Configuration file options

Input and output options

PDF parameters

Other parameters

The plotdata program

Selecting files

Selecting x and y data

Examples

Quick-start guide

pdfgetx3 command

pdfgetn3 command

pdfgets3 command

Release notes

2.4.0

2.3.0

Release notes

2.2.2

2.2.1

2.2.0

2.1.2

2.1.1

2.1.0

2.0.0

1.2

Tutorial

Nickel X-ray PDF

predefined configuration file

processing from scratch

Platinum X-ray series

matching input files

data search path

output file names

Interactive tuning of parameters

Neutron PDF

Fit twothetazero

Small-angle-scattering PDF (sasPDF)

Multiple Background Subtraction

Predefined configuration file

Interactive Multiple Background Subtraction