The new uses.py tool is the first stage of looking for
authorDavid Anderson <davea42@earthlink.net>
Sun, 3 Jun 2012 18:17:46 +0000 (11:17 -0700)
committerDavid Anderson <davea42@earthlink.net>
Sun, 3 Jun 2012 18:17:46 +0000 (11:17 -0700)
and finding questionable entries in the document without
having to interpret latex messages.

Duplicate and missing labels etc.
This creates the reader and modifies all so we continue
to reuse fileio.py everywhere.  With regression tests
included.

No change to the document .tex files.

dwarf5/tools/Makefile
dwarf5/tools/anylink.py
dwarf5/tools/fileio.py
dwarf5/tools/printstandard.py
dwarf5/tools/refclassfixup.py
dwarf5/tools/tohyphen.py
dwarf5/tools/uses.base [new file with mode: 0644]
dwarf5/tools/uses.in [new file with mode: 0644]
dwarf5/tools/uses.py [new file with mode: 0644]
dwarf5/tools/usesb.base [new file with mode: 0644]
dwarf5/tools/usesb.in [new file with mode: 0644]

index c2af124..3933d57 100644 (file)
@@ -56,4 +56,11 @@ test:
        diff anylinkt.in.out anylinkt.base
        python anylink.py -t DW_ACCESS_ -t DW_OP_ anylinkt.in.out
        diff anylinkt.in.out.out anylinkt.base
+       #
+       python uses.py usesb.in >usesb.in.out 
+       diff usesb.in.out usesb.base
+       #
+       python uses.py uses.in >uses.in.out 
+       diff uses.in.out uses.base
+
      
index d2b5783..5acffd4 100644 (file)
@@ -91,7 +91,7 @@ def trailerisrbrace(linetoks,num):
     return "n"
   return "n"
 
-def transfunc(linetoks):
+def transfunc(linetoks,myfile,linenum):
   if len(linetoks) < 1:
     return linetoks
   tnumin = 0
index b885c65..167ad45 100644 (file)
@@ -255,8 +255,8 @@ class  dwline:
     for t in self._toks:
       t.dwwrite(outfile)
     outfile.write("\n")
-  def dwtransformline(self,callfunc):
-    toks = callfunc(self._toks)
+  def dwtransformline(self,callfunc,myfile,lnum):
+    toks = callfunc(self._toks,myfile,lnum)
     self._toks = toks
     
 
@@ -304,9 +304,11 @@ class dwfile:
     for l in self._lines:
       l.dwwrite(outfile,lnum)
       lnum = lnum + 1
-  def dwtransformline(self,callfunc):
+  def dwtransformline(self,callfunc,myfile):
+    lnum=1
     for l in self._lines:
-      l.dwtransformline(callfunc)
+      l.dwtransformline(callfunc,myfile,lnum)
+      lnum = lnum + 1
     
 
 
@@ -328,7 +330,7 @@ class dwfiles:
       f.dwwrite()
   def dwtransformline(self,callfunc):
     for f in self._files:
-      f.dwtransformline(callfunc)
+      f.dwtransformline(callfunc,f)
 
 
 def readFilelist(filelist):
index 3e5620e..c7e376a 100644 (file)
@@ -8,7 +8,7 @@
 import sys
 import fileio
 
-def transfunc(linetoks):
+def transfunc(linetoks,myfile,linenum):
   if len(linetoks) < 1:
     return linetoks
   tnumin = 0
index e8913b4..ccf8dcd 100644 (file)
@@ -49,7 +49,7 @@ def append_to_out(out,addthese):
   for a in addthese:
     out += [a]
 
-def transfunc(linetoks):
+def transfunc(linetoks,myfile,linenum):
   if len(linetoks) < 1:
     return linetoks
   tnumin = 0
index 58e523f..b0d8c96 100644 (file)
@@ -17,7 +17,7 @@ def convertToHyphen(s):
   os = ''.join(out)
   return os
 
-def transfunc(linetoks):
+def transfunc(linetoks,myfile,linenum):
   if len(linetoks) < 1:
     return linetoks
   outtoks = []
diff --git a/dwarf5/tools/uses.base b/dwarf5/tools/uses.base
new file mode 100644 (file)
index 0000000..ba33668
--- /dev/null
@@ -0,0 +1,16 @@
+ERROR insufficient tokens on line for pattern  i { i } { i } { * }  line  6  file  uses.in
+ERROR insufficient tokens on line for pattern  i { i } { i } { * }  line  17  file  uses.in
+ERROR insufficient tokens on line for pattern  i { i } { i } { * }  line  18  file  uses.in
+ERROR insufficient tokens on line for pattern  i { i } { i } { * }  line  21  file  uses.in
+ERROR line  expected }  got  ['t', 'o'] pattern i { i } { i }  line  27  file  uses.in
+ERROR line  expected }  got  ['n', 'o', 't'] pattern i { i } { i }  line  30  file  uses.in
+ERROR line  expected }  got  ['n', 'o', 't'] pattern i { i } { i } { * }  line  31  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i }  line  36  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i } { * }  line  37  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i }  line  40  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i }  line  41  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i }  line  50  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i } { * }  line  51  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i }  line  54  file  uses.in
+ERROR line  expected }  got  ['l', 'i', 'n', 'k', 'e', 'd'] pattern i { i } { i }  line  55  file  uses.in
+ERROR line  expected }  got  ['{'] pattern i { i } { i }  line  67  file  uses.in
diff --git a/dwarf5/tools/uses.in b/dwarf5/tools/uses.in
new file mode 100644 (file)
index 0000000..6735cc7
--- /dev/null
@@ -0,0 +1,74 @@
+
+\livelink{a}{linkahasdef}
+\livetarg{a}{linkahasdef}
+
+\livelink{b}{linkbhasdef}
+\livetargi{b}{linkbhasdef}{descr linkb}
+
+\refersec{chap:dwithdef}
+\label{chap:dwithdef}
+
+
+ Following is error
+\livetarg{multdefa}{multdefa}
+\livetarg{multdefa}{multdefa}
+
+ Following is error
+\livetargi{multdefb}{multdefb}{mult def b}
+\livetargi{multdefb}{multdefb}{mult def b}
+
+\livetarg{multdefc}{multdefc}
+\livetargi{multdefc}{multdefc}{mult def b}
+
+Following is ref to nonexistent def
+\refersec{chap:notdefineda}
+
+Following is link to nonexistent targ
+\livelink{linktoundefined}{link to undefined}
+
+Following are targs with no links or labels
+\livetarg{targnotlinkedtoa}{ targ not linked to a}
+\livetargi{targnotlinkedtob}{ targ not linked to b}
+\refersec{chap:unresolvedref}
+
+
+Following are targs with single links or labels
+\livetarg{targlinkedaa}{ targ linked once}
+\livetargi{targlinkedbb}{ targ linked once}{ targ linked once }
+\label{chap:resolvedrefcc}
+
+\livelink{targlinkedbb}{ targ linked once}
+\livelink{targlinkedaa}{ targ linked once}
+\refersec{chap:resolvedrefcc}
+
+Following is not in a macro, so likely an oversight.
+DW_TAG_mystuffprobablyoversight
+
+
+
+Following are targs with two links or labels
+\livetarg{targlinkeddd}{ targ linked twice}
+\livetargi{targlinkedee}{ targ linked twice}{ targ linked twice }
+\label{chap:resolvedrefff}
+
+\livelink{targlinkedbb}{ targ linked once}
+\livelink{targlinkedaa}{ targ linked once}
+\refersec{chap:resolvedrefff}
+
+# Isses with DW names and target mismatch. Typos, usually.
+\livetarg{chap:DWTAGfoo}{DW\_TAG\_fool}
+\livetarg{chap:DWTAGfooj}{DW\-\_TAG\-\_foolj}
+\livelink{chap:DWTAGfoo}{DW\_TAG\_fool}
+\livelink{chap:DWTAGfooj}{DW\-\_TAG\-\_foolj}
+#name mismatch and missing link targ
+\livelink{chap:DWTAGfook}{DW\-\_TAG\-\_foolk}
+
+# valid, single ref, no DW misspell.
+\livetarg{chap:DWTAGfooe{DW\_TAG\_fooe}
+\livetarg{chap:DWTAGfoog}{DW\-\_TAG\-\_foog}
+\livelink{chap:DWTAGfooe}{DW\_TAG\_fooe}
+\livelink{chap:DWTAGfoog}{DW\-\_TAG\-\_foog}
+
+
+
+
diff --git a/dwarf5/tools/uses.py b/dwarf5/tools/uses.py
new file mode 100644 (file)
index 0000000..ce8ad92
--- /dev/null
@@ -0,0 +1,331 @@
+# Copyright 2012 DWARF Debugging Information Format Committee
+#
+# Handles the testing and update for all DW_* prefixes.
+# Called by taglink and other convenience apps to do their work.
+#
+# Run as an app itself, the options are
+#    python anylink [-t prefix] ... [-all] [file] ...
+#    Use either -all or one or more -t, as in examples:
+#    python anylink -t DW_ACCESS_ -t DW_OP_   test.in test2.in
+#    python anylink -all    test.in test2.in 
+
+import sys
+import fileio
+
+global linkdefinitionsdict
+global linkusesdict
+global labeldefinitionsdict
+global labelusesdict
+global ignorethesedict
+global indexsetdict
+global dupdefcount
+global unresolveddwdict
+# Links meaning \livelink \livetarg \livetargi macros
+linkdefinitionsdict = {}
+linkusesdict  = {}
+# labels meaning \refersec (a ref) and \label  (a def)
+labeldefinitionsdict = {}
+labelusesdict =  {}
+# The dict of indexed things.
+indexsetdict ={}
+# DW sorts of names not sensibly resolved.
+unresolveddwdict = {}
+dupdefcount = 0
+
+
+
+# a list of words to ignore: silly stuff.
+ignorethesedict = {"of":0, "a":0, "the":0, "and":0, "but":0,"DWARF":0,
+"Standards":0,"Committee":0,"Version":0 }
+
+class tokmention:
+  def __init__(self):
+    self._token = '' 
+    self._file = ""
+    self._line = 0
+    # Class is "id", "ind","other","none"
+  def __init__(self,tok,filename,line):
+    self._token = tok
+    self._file = filename
+    self._line = line
+
+
+
+
+def ischar(tok,c):
+   if tok._class != "ind":
+      return "n"
+   if len(tok._tex) != 1:
+       return "n"
+   if tok._tex[0] != c:
+       return "n"
+   return "y"
+
+def dwspace(tok):
+  if ischar(tok," ") == "y":
+    return "y"
+  if ischar(tok,"\t") == "y":
+    return "y"
+  return "n"
+  
+  
+def isbrace(tok,brace):
+  if tok._class != "ind":
+     return "n"
+  if len(tok._tex) != 1:
+     return "n"
+  if brace == tok._tex[0]:
+     return "y"
+  return "n"
+
+
+def pickup(linetoks,tnumin,pattern,myfile,linenum):
+  """ The token pattern characters are
+  i meaning identifier
+  [space] meaning whitespace
+  { meaning left brace
+  } meaning right brace
+  * meaning any token except } and end-line
+  
+  Precondition:  linetoks[tnumin] is identifier (meaning a command)
+  Returns: a token list, one per non-space in the pattern.
+     For the *, the token is itself a list of whatever it contains.
+  """
+  outtoks = []
+  numabsorbed = 1
+  inlen = len(linetoks) 
+  curnum = tnumin
+  curtok = linetoks[curnum]
+  outtoks += [curtok]
+  patterncharnum = -1
+  for c in pattern:
+    patterncharnum = patterncharnum + 1
+    if curnum >= inlen:
+      print "ERROR line ended surprisingly, pattern ", pattern,"  line ",linenum," file ",myfile._name
+      return outtoks,numabsorbed
+    curtok = linetoks[curnum]
+    if c == " ":
+      while dwspace(curtok) == "y":
+        curnum = curnum + 1
+        if curnum >= inlen:
+          print "ERROR line ended surprisingly in space, pattern ", pattern, " line ",linenum," file ",myfile._name
+          return outtoks,numabsorbed
+        numabsorbed = numabsorbed + 1
+        curtok = linetoks[curnum]
+      continue
+    elif c == "i":
+      if curtok._class != "id":
+        print "ERROR line  expected identifier got ",curtok._tex, "pattern" , pattern, " line " ,linenum," file ",myfile._name
+        return outtoks,numabsorbed
+      numabsorbed = numabsorbed + 1
+      outtoks += [curtok]
+      curnum = curnum + 1
+      continue
+    elif c == "{":
+      if isbrace(curtok,"{")  == "y":
+        outtoks += [curtok]
+        curnum = curnum + 1
+        numabsorbed = numabsorbed + 1
+      else:
+        print "ERROR line  expected {  got ",curtok._tex," pattern ",pattern," line " ,linenum," file ",myfile._name
+        return outtoks,numabsorbed
+    elif c == "}":
+      if isbrace(curtok,"}")  == "y":
+        outtoks += [curtok]
+        curnum = curnum + 1
+        numabsorbed = numabsorbed + 1
+      else:
+        print "ERROR line  expected }  got ",curtok._tex,"pattern",pattern," line " ,linenum," file ",myfile._name
+        return outtoks,numabsorbed
+    elif c == "*":
+      outlist = []
+      while isbrace(curtok,"}") == "n":
+        curtok = linetoks[curnum]
+        if dwspace(curtok) == "n":
+           outlist += [curtok]
+        curnum = curnum + 1
+        if curnum >= inlen:
+          outtoks += [outlist]
+          if patterncharnum < (len(pattern) -1): 
+            print "ERROR insufficient tokens on line for pattern ", pattern," line " ,linenum," file ",myfile._name
+          return outtoks,numabsorbed
+        numabsorbed = numabsorbed + 1
+      # Found a right brace, so done here.
+      outtoks += [outlist]
+    else:
+        print "ERROR pattern had unexpected character ",pattern
+        sys.exit(1)
+  return outtoks,numabsorbed
+
+def reftodict(d,k,v):
+  keystring = ''.join(k._token._tex)
+  if d.has_key(keystring) == 0:
+     d[keystring] =  [v]
+  else:
+     existing = d.get(keystring)
+     existing += [v]
+     d[keystring] =  existing
+
+def deftodict(d,k,v):
+  global dupdefcount
+  keystring = ''.join(k._token._tex)
+  if d.has_key(keystring) == 0:
+     d[keystring] =  [v]
+  else:
+     # This is a duplication, we just record it here,
+     # we will report on it shortly.
+     dupdefcount = dupdefcount + 1
+     existing = d.get(keystring)
+     existing += [v]
+     d[keystring] =  existing
+
+def livetargprocess(linetoks,tnumin,myfile,linenum):
+  """ \livetarg{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
+  global linkdefinitionsdict
+  global linkusesdict
+  global indexsetdict
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { i }",myfile,linenum)
+  if len(ourtoks) > 5:
+    targlink= tokmention(ourtoks[2],myfile,linenum)
+    targname= tokmention(ourtoks[5],myfile,linenum)
+    deftodict(linkdefinitionsdict,targlink,targname)
+    reftodict(indexsetdict,targname,targname)
+  return inlen
+def livetargiprocess(linetoks,tnumin,myfile,linenum):
+  """ \livetargi{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter}{name of targ} """
+  global linkdefinitionsdict
+  global linkusesdict
+  global indexsetdict
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { i } { * }",myfile,linenum)
+  if len(ourtoks) > 5:
+    targlink= tokmention(ourtoks[2],myfile,linenum)
+    targname= tokmention(ourtoks[5],myfile,linenum)
+    deftodict(linkdefinitionsdict,targlink,targname)
+    reftodict(indexsetdict,targname,targname)
+  return inlen
+def livelinkprocess(linetoks,tnumin,myfile,linenum):
+  """ \livelink{chap:DWTAGtemplatevalueparameter}{DW\-\_TAG\-\_template\-\_value\-\_parameter} """
+  global linkdefinitionsdict
+  global linkusesdict
+  global indexsetdict
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i } { i }",myfile,linenum)
+  if len(ourtoks) > 5:
+    targlink= tokmention(ourtoks[2],myfile,linenum)
+    targname= tokmention(ourtoks[5],myfile,linenum)
+    reftodict(linkusesdict,targlink,targname)
+    reftodict(indexsetdict,targname,targname)
+  return inlen
+def labelprocess(linetoks,tnumin,myfile,linenum):
+  """ \label{alabel} """
+  global labeldefinitionsdict
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
+  if len(ourtoks) > 2:
+    label = tokmention(ourtoks[2],myfile,linenum)
+    deftodict(labeldefinitionsdict,label,label)
+  return inlen
+def addtoindexprocess(linetoks,tnumin,myfile,linenum):
+  """ \addtoindex{alabel} """
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
+  if len(ourtoks) > 2:
+    index = tokmention(ourtoks[2],myfile,linenum)
+    reftodict(indexsetdict,index,index)
+  return inlen
+def indexprocess(linetoks,tnumin,myfile,linenum):
+  """ \index{indexentryname} """
+  global labelusesdict
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
+  if len(ourtoks) > 2:
+    myentry = tokmention(ourtoks[2],myfile,linenum)
+    reftodict(indexsetdict,myentry,myentry)
+  return inlen
+def refersecprocess(linetoks,tnumin,myfile,linenum):
+  """ \refersec{label} """
+  global labelusesdict
+  t = linetoks[tnumin]
+  ourtoks,inlen = pickup(linetoks,tnumin,"i { i }",myfile,linenum)
+  if len(ourtoks) > 2:
+    label = tokmention(ourtoks[2],myfile,linenum)
+    reftodict(labelusesdict,label,label)
+  return inlen
+
+def transfunc(linetoks,myfile,linenum):
+  if len(linetoks) < 1:
+    return linetoks
+  tnumin = 0
+  lasttoknum = len(linetoks)
+  while tnumin < lasttoknum:
+    t = linetoks[tnumin]
+    tnumcount = 1
+    rawtok = ''.join(t._tex)
+    stdname= ''.join(t._std)
+    if rawtok == "\\livetarg":
+      tnumcount = livetargprocess(linetoks,tnumin,myfile,linenum)
+    elif rawtok == "\\livetargi":
+      tnumcount = livetargiprocess(linetoks,tnumin,myfile,linenum)
+    elif rawtok == "\\livelink":
+      tnumcount = livelinkprocess(linetoks,tnumin,myfile,linenum)
+    elif rawtok == "\\label":
+      tnumcount = labelprocess(linetoks,tnumin,myfile,linenum)
+    elif rawtok == "\\refersec":
+      tnumcount = refersecprocess(linetoks,tnumin,myfile,linenum)
+    elif rawtok == "\\addtoindex":
+      tnumcount = addtoindexprocess(linetoks,tnumin,myfile,linenum)
+    elif rawtok == "\\index":
+      tnumcount = indexprocess(linetoks,tnumin,myfile,linenum)
+    else:  
+      if t._class == "id":
+        namemention = tokmention(t,myfile,linenum)
+        namestring = ''.join(t._std)
+        if namestring.startswith("DW"):
+          global unresolveddwdict
+          reftodict(unresolveddwdict,namemention,namemention)
+        # Else we might want to build a dict of all other words 
+        # while leaving out all \latex commands we don't know?
+    tnumin = tnumin + tnumcount
+    # End of for loop.
+  return linetoks
+
+def process_files(filelist):
+  dwf = fileio.readFilelist(filelist)
+  # We will really just report, not transform
+  # anything, but this works.
+  dwf.dwtransformline(transfunc)
+
+  # Here we report on our discoveries.
+
+def read_file_args(targlist):
+  cur = 1
+  filelist = []
+  while  len(sys.argv) > cur:
+    v = sys.argv[cur]
+    filelist += [v]
+    cur = int(cur) + 1
+  process_files(filelist)
+  
+
+
+
+def read_all_args():
+  filelist = []
+  cur = 1
+  while  len(sys.argv) > cur:
+    v = sys.argv[cur]
+    filelist += [v]
+    cur = int(cur) + 1
+  if len(filelist) < 1:
+    print >> sys.stderr , "No files specified."
+    printlegals()
+    sys.exit(1)
+  process_files(filelist)
+
+#  anylink [-t <class>] ... [file] ...
+
+if __name__ == '__main__':
+  read_all_args()
+
diff --git a/dwarf5/tools/usesb.base b/dwarf5/tools/usesb.base
new file mode 100644 (file)
index 0000000..0a04273
--- /dev/null
@@ -0,0 +1,6 @@
+ERROR line ended surprisingly in space, pattern  i { i } { i }  line  2  file  usesb.in
+ERROR insufficient tokens on line for pattern  i { i } { i } { * }  line  3  file  usesb.in
+ERROR line ended surprisingly, pattern  i { i } { i }   line  4  file  usesb.in
+ERROR line ended surprisingly, pattern  i { i }   line  5  file  usesb.in
+ERROR line  expected }  got  ['{'] pattern i { i } { i }  line  6  file  usesb.in
+ERROR line  expected {  got  ['}']  pattern  i { i } { i }  line  7  file  usesb.in
diff --git a/dwarf5/tools/usesb.in b/dwarf5/tools/usesb.in
new file mode 100644 (file)
index 0000000..678ee93
--- /dev/null
@@ -0,0 +1,10 @@
+\livetarg { a } { linkahasdef    } \livetarg { x } { linkhasdef    }
+\livetarg { c } { linkchasdef 
+\livetargi {b} {f} {   a b c 
+\livetarg
+\index   {
+\livetarg { af { { linkahasdef    } \livetarg { x } { linkhasdef    }
+\livetarg { aj } } linkahasdef    } \livetarg { x } { linkhasdef    }
+
+
+